languages/cpython
languages/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython synced 2024-09-16 02:00:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Marc-Andre's third try at this bulk patch seems to work (except that

Browse source 
his copy of test_contains.py seems to be broken -- the lines he
deleted were already absent).  Checkin messages:


New Unicode support for int(), float(), complex() and long().

- new APIs PyInt_FromUnicode() and PyLong_FromUnicode()
- added support for Unicode to PyFloat_FromString()
- new encoding API PyUnicode_EncodeDecimal() which converts
  Unicode to a decimal char* string (used in the above new
  APIs)
- shortcuts for calls like int(<int object>) and float(<float obj>)
- tests for all of the above

Unicode compares and contains checks:
- comparing Unicode and non-string types now works; TypeErrors
  are masked, all other errors such as ValueError during
  Unicode coercion are passed through (note that PyUnicode_Compare
  does not implement the masking -- PyObject_Compare does this)
- contains now works for non-string types too; TypeErrors are
  masked and 0 returned; all other errors are passed through

Better testing support for the standard codecs.

Misc minor enhancements, such as an alias dbcs for the mbcs codec.

Changes:
- PyLong_FromString() now applies the same error checks as
  does PyInt_FromString(): trailing garbage is reported
  as error and not longer silently ignored. The only characters
  which may be trailing the digits are 'L' and 'l' -- these
  are still silently ignored.
- string.ato?() now directly interface to int(), long() and
  float(). The error strings are now a little different, but
  the type still remains the same. These functions are now
  ready to get declared obsolete ;-)
- PyNumber_Int() now also does a check for embedded NULL chars
  in the input string; PyNumber_Long() already did this (and
  still does)

Followed by:

Looks like I've gone a step too far there... (and test_contains.py
seem to have a bug too).

I've changed back to reporting all errors in PyUnicode_Contains()
and added a few more test cases to test_contains.py (plus corrected
the join() NameError).
This commit is contained in:
Guido van Rossum 2000-04-05 20:11:21 +00:00
parent 457855a5f0
commit 9e896b37c7
17 changed files with 421 additions and 115 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Include/Python.h
View file

@ -72,6 +72,7 @@ PERFORMANCE OF THIS SOFTWARE.
#include "pydebug.h"
#include "unicodeobject.h"
#include "intobject.h"
#include "longobject.h"
#include "floatobject.h"
@ -92,7 +93,6 @@ PERFORMANCE OF THIS SOFTWARE.
#include "cobject.h"
#include "traceback.h"
#include "sliceobject.h"
#include "unicodeobject.h"
#include "codecs.h"
#include "pyerrors.h"

1
Include/intobject.h
View file

@ -60,6 +60,7 @@ extern DL_IMPORT(PyTypeObject) PyInt_Type;
#define PyInt_Check(op) ((op)->ob_type == &PyInt_Type)
extern DL_IMPORT(PyObject *) PyInt_FromString Py_PROTO((char*, char**, int));
extern DL_IMPORT(PyObject *) PyInt_FromUnicode Py_PROTO((Py_UNICODE*, int, int));
extern DL_IMPORT(PyObject *) PyInt_FromLong Py_PROTO((long));
extern DL_IMPORT(long) PyInt_AsLong Py_PROTO((PyObject *));
extern DL_IMPORT(long) PyInt_GetMax Py_PROTO((void));

1
Include/longobject.h
View file

@ -82,6 +82,7 @@ extern DL_IMPORT(unsigned LONG_LONG) PyLong_AsUnsignedLongLong Py_PROTO((PyObjec
#endif /* HAVE_LONG_LONG */
DL_IMPORT(PyObject *) PyLong_FromString Py_PROTO((char *, char **, int));
DL_IMPORT(PyObject *) PyLong_FromUnicode Py_PROTO((Py_UNICODE*, int, int));
#ifdef __cplusplus
}

35
Include/unicodeobject.h
View file

@ -358,7 +358,7 @@ extern DL_IMPORT(PyObject*) PyUnicode_EncodeUTF8(
/* --- UTF-16 Codecs ------------------------------------------------------ */
/* Decodes length bytes from a UTF-16 encoded buffer string and return
/* Decodes length bytes from a UTF-16 encoded buffer string and returns
the corresponding Unicode object.
errors (if non-NULL) defines the error handling. It defaults
@ -397,7 +397,7 @@ extern DL_IMPORT(PyObject*) PyUnicode_AsUTF16String(
);
/* Returns a Python string object holding the UTF-16 encoded value of
the Unicode data in s.
the Unicode data.
If byteorder is not 0, output is written according to the following
byte order:
@ -587,6 +587,37 @@ extern DL_IMPORT(PyObject*) PyUnicode_EncodeMBCS(
#endif /* MS_WIN32 */
/* --- Decimal Encoder ---------------------------------------------------- */
/* Takes a Unicode string holding a decimal value and writes it into
an output buffer using standard ASCII digit codes.
The output buffer has to provide at least length+1 bytes of storage
area. The output string is 0-terminated.
The encoder converts whitespace to ' ', decimal characters to their
corresponding ASCII digit and all other Latin-1 characters except
\0 as-is. Characters outside this range (Unicode ordinals 1-256)
are treated as errors. This includes embedded NULL bytes.
Error handling is defined by the errors argument:
NULL or "strict": raise a ValueError
"ignore": ignore the wrong characters (these are not copied to the
output buffer)
"replace": replaces illegal characters with '?'
Returns 0 on success, -1 on failure.
*/
extern DL_IMPORT(int) PyUnicode_EncodeDecimal(
Py_UNICODE *s, /* Unicode buffer */
int length, /* Number of Py_UNICODE chars to encode */
char *output, /* Output buffer; must have size >= length */
const char *errors /* error handling */
);
/* --- Methods & Slots ----------------------------------------------------
These are capable of handling Unicode objects and strings on input

6
Lib/encodings/__init__.py
View file

@ -4,8 +4,8 @@
directory.
Codec modules must have names corresponding to standard lower-case
encoding names. Hyphens are automatically converted to
underscores, e.g. 'utf-8' is looked up as module utf_8.
encoding names with hyphens mapped to underscores, e.g. 'utf-8' is
implemented by the module 'utf_8.py'.
Each codec module must export the following interface:
@ -40,7 +40,7 @@ def search_function(encoding):
return entry
# Import the module
modname = string.replace(encoding,'-','_')
modname = string.replace(encoding, '-', '_')
modname = aliases.aliases.get(modname,modname)
try:
mod = __import__(modname,globals(),locals(),'*')

3
Lib/encodings/aliases.py
View file

@ -54,4 +54,7 @@
'macroman': 'mac_roman',
'macturkish': 'mac_turkish',
# MBCS
'dbcs': 'mbcs',
}

39
Lib/string.py
View file

@ -196,14 +196,11 @@ def atof(s):
Return the floating point number represented by the string s.
"""
if type(s) == _StringType:
return _float(s)
else:
raise TypeError('argument 1: expected string, %s found' %
type(s).__name__)
return _float(s)
# Convert string to integer
def atoi(*args):
def atoi(s , base=10):
"""atoi(s [,base]) -> int
Return the integer represented by the string s in the given
@ -214,23 +211,11 @@ def atoi(*args):
accepted.
"""
try:
s = args[0]
except IndexError:
raise TypeError('function requires at least 1 argument: %d given' %
len(args))
# Don't catch type error resulting from too many arguments to int(). The
# error message isn't compatible but the error type is, and this function
# is complicated enough already.
if type(s) == _StringType:
return _apply(_int, args)
else:
raise TypeError('argument 1: expected string, %s found' %
type(s).__name__)
return _int(s, base)
# Convert string to long integer
def atol(*args):
def atol(s, base=10):
"""atol(s [,base]) -> long
Return the long integer represented by the string s in the
@ -242,19 +227,7 @@ def atol(*args):
unless base is 0.
"""
try:
s = args[0]
except IndexError:
raise TypeError('function requires at least 1 argument: %d given' %
len(args))
# Don't catch type error resulting from too many arguments to long(). The
# error message isn't compatible but the error type is, and this function
# is complicated enough already.
if type(s) == _StringType:
return _apply(_long, args)
else:
raise TypeError('argument 1: expected string, %s found' %
type(s).__name__)
return _long(s, base)
# Left-justify a string

1
Lib/test/output/test_unicode
View file

@ -3,3 +3,4 @@ Testing Unicode comparisons... done.
Testing Unicode contains method... done.
Testing Unicode formatting strings... done.
Testing builtin codecs... done.
Testing standard mapping codecs... 0-127... 128-255... done.

37
Lib/test/test_b1.py
View file

@ -95,6 +95,7 @@ def __call__(self): pass
if complex(0j, 3.14) <> 3.14j: raise TestFailed, 'complex(0j, 3.14)'
if complex(0.0, 3.14) <> 3.14j: raise TestFailed, 'complex(0.0, 3.14)'
if complex(" 3.14+J ") <> 3.14+1j: raise TestFailed, 'complex(" 3.14+J )"'
if complex(u" 3.14+J ") <> 3.14+1j: raise TestFailed, 'complex(u" 3.14+J )"'
class Z:
def __complex__(self): return 3.14j
z = Z()
@ -208,6 +209,9 @@ def identity(item):
if float(314) <> 314.0: raise TestFailed, 'float(314)'
if float(314L) <> 314.0: raise TestFailed, 'float(314L)'
if float(" 3.14 ") <> 3.14: raise TestFailed, 'float(" 3.14 ")'
if float(u" 3.14 ") <> 3.14: raise TestFailed, 'float(u" 3.14 ")'
if float(u" \u0663.\u0661\u0664 ") <> 3.14:
raise TestFailed, 'float(u" \u0663.\u0661\u0664 ")'
print 'getattr'
import sys
@ -254,6 +258,9 @@ def f(): pass
if int(-3.9) <> -3: raise TestFailed, 'int(-3.9)'
if int(3.5) <> 3: raise TestFailed, 'int(3.5)'
if int(-3.5) <> -3: raise TestFailed, 'int(-3.5)'
# Different base:
if int("10",16) <> 16L: raise TestFailed, 'int("10",16)'
if int(u"10",16) <> 16L: raise TestFailed, 'int(u"10",16)'
# Test conversion fron strings and various anomalies
L = [
('0', 0),
@ -267,9 +274,28 @@ def f(): pass
('314 ', 314),
(' \t\t 314 \t\t ', 314),
(`sys.maxint`, sys.maxint),
(' 1x', ValueError),
(' 1 ', 1),
(' 1\02 ', ValueError),
('', ValueError),
(' ', ValueError),
(' \t\t ', ValueError),
(u'0', 0),
(u'1', 1),
(u'9', 9),
(u'10', 10),
(u'99', 99),
(u'100', 100),
(u'314', 314),
(u' 314', 314),
(u'\u0663\u0661\u0664 ', 314),
(u' \t\t 314 \t\t ', 314),
(u' 1x', ValueError),
(u' 1 ', 1),
(u' 1\02 ', ValueError),
(u'', ValueError),
(u' ', ValueError),
(u' \t\t ', ValueError),
]
for s, v in L:
for sign in "", "+", "-":
@ -349,10 +375,17 @@ class E:
if long(-3.9) <> -3L: raise TestFailed, 'long(-3.9)'
if long(3.5) <> 3L: raise TestFailed, 'long(3.5)'
if long(-3.5) <> -3L: raise TestFailed, 'long(-3.5)'
if long("-3") <> -3L: raise TestFailed, 'long("-3")'
if long(u"-3") <> -3L: raise TestFailed, 'long(u"-3")'
# Different base:
if long("10",16) <> 16L: raise TestFailed, 'long("10",16)'
if long(u"10",16) <> 16L: raise TestFailed, 'long(u"10",16)'
# Check conversions from string (same test set as for int(), and then some)
LL = [
('1' + '0'*20, 10L**20),
('1' + '0'*100, 10L**100),
(u'1' + u'0'*20, 10L**20),
(u'1' + u'0'*100, 10L**100),
]
for s, v in L + LL:
for sign in "", "+", "-":
@ -363,11 +396,11 @@ class E:
vv = -v
try:
if long(ss) != long(vv):
raise TestFailed, "int(%s)" % `ss`
raise TestFailed, "long(%s)" % `ss`
except v:
pass
except ValueError, e:
raise TestFailed, "int(%s) raised ValueError: %s" % (`ss`, e)
raise TestFailed, "long(%s) raised ValueError: %s" % (`ss`, e)
print 'map'
if map(None, 'hello world') <> ['h','e','l','l','o',' ','w','o','r','l','d']:

101
Lib/test/test_unicode.py
View file

@ -221,15 +221,23 @@ def __init__(self): self.seq = [7, u'hello', 123L]
# Contains:
print 'Testing Unicode contains method...',
assert ('a' in 'abdb') == 1
assert ('a' in 'bdab') == 1
assert ('a' in 'bdaba') == 1
assert ('a' in 'bdba') == 1
assert ('a' in u'abdb') == 1
assert ('a' in u'bdab') == 1
assert ('a' in u'bdaba') == 1
assert ('a' in u'bdba') == 1
assert ('a' in u'bdba') == 1
assert (u'a' in u'bdba') == 1
assert (u'a' in u'bdb') == 0
assert (u'a' in 'bdb') == 0
assert (u'a' in 'bdba') == 1
assert (u'a' in ('a',1,None)) == 1
assert (u'a' in (1,None,'a')) == 1
assert (u'a' in (1,None,u'a')) == 1
assert ('a' in ('a',1,None)) == 1
assert ('a' in (1,None,'a')) == 1
assert ('a' in (1,None,u'a')) == 1
assert ('a' in ('x',1,u'y')) == 0
assert ('a' in ('x',1,None)) == 0
print 'done.'
# Formatting:
@ -270,11 +278,88 @@ def __init__(self): self.seq = [7, u'hello', 123L]
assert unicode(u.encode(encoding),encoding) == u
u = u''.join(map(unichr, range(256)))
for encoding in ('latin-1',):
assert unicode(u.encode(encoding),encoding) == u
for encoding in (
'latin-1',
):
try:
assert unicode(u.encode(encoding),encoding) == u
except AssertionError:
print '*** codec "%s" failed round-trip' % encoding
except ValueError,why:
print '*** codec for "%s" failed: %s' % (encoding, why)
u = u''.join(map(unichr, range(128)))
for encoding in ('ascii',):
assert unicode(u.encode(encoding),encoding) == u
for encoding in (
'ascii',
):
try:
assert unicode(u.encode(encoding),encoding) == u
except AssertionError:
print '*** codec "%s" failed round-trip' % encoding
except ValueError,why:
print '*** codec for "%s" failed: %s' % (encoding, why)
print 'done.'
print 'Testing standard mapping codecs...',
print '0-127...',
s = ''.join(map(chr, range(128)))
for encoding in (
'cp037', 'cp1026',
'cp437', 'cp500', 'cp737', 'cp775', 'cp850',
'cp852', 'cp855', 'cp860', 'cp861', 'cp862',
'cp863', 'cp865', 'cp866',
'iso8859_10', 'iso8859_13', 'iso8859_14', 'iso8859_15',
'iso8859_2', 'iso8859_3', 'iso8859_4', 'iso8859_5', 'iso8859_6',
'iso8859_7', 'iso8859_9', 'koi8_r', 'latin_1',
'mac_cyrillic', 'mac_latin2',
'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255',
'cp1256', 'cp1257', 'cp1258',
'cp856', 'cp857', 'cp864', 'cp869', 'cp874',
'mac_greek', 'mac_iceland','mac_roman', 'mac_turkish',
'cp1006', 'cp875', 'iso8859_8',
### These have undefined mappings:
#'cp424',
):
try:
assert unicode(s,encoding).encode(encoding) == s
except AssertionError:
print '*** codec "%s" failed round-trip' % encoding
except ValueError,why:
print '*** codec for "%s" failed: %s' % (encoding, why)
print '128-255...',
s = ''.join(map(chr, range(128,256)))
for encoding in (
'cp037', 'cp1026',
'cp437', 'cp500', 'cp737', 'cp775', 'cp850',
'cp852', 'cp855', 'cp860', 'cp861', 'cp862',
'cp863', 'cp865', 'cp866',
'iso8859_10', 'iso8859_13', 'iso8859_14', 'iso8859_15',
'iso8859_2', 'iso8859_3', 'iso8859_4', 'iso8859_5', 'iso8859_6',
'iso8859_7', 'iso8859_9', 'koi8_r', 'latin_1',
'mac_cyrillic', 'mac_latin2',
### These have undefined mappings:
#'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255',
#'cp1256', 'cp1257', 'cp1258',
#'cp424', 'cp856', 'cp857', 'cp864', 'cp869', 'cp874',
#'mac_greek', 'mac_iceland','mac_roman', 'mac_turkish',
### These fail the round-trip:
#'cp1006', 'cp875', 'iso8859_8',
):
try:
assert unicode(s,encoding).encode(encoding) == s
except AssertionError:
print '*** codec "%s" failed round-trip' % encoding
except ValueError,why:
print '*** codec for "%s" failed: %s' % (encoding, why)
print 'done.'

91
Objects/abstract.c
View file

@ -726,6 +726,27 @@ PyNumber_Absolute(o)
return type_error("bad operand type for abs()");
}
/* Add a check for embedded NULL-bytes in the argument. */
static PyObject *
int_from_string(s, len)
const char *s;
int len;
{
char *end;
PyObject *x;
x = PyInt_FromString((char*)s, &end, 10);
if (x == NULL)
return NULL;
if (end != s + len) {
PyErr_SetString(PyExc_ValueError,
"null byte in argument for int()");
Py_DECREF(x);
return NULL;
}
return x;
}
PyObject *
PyNumber_Int(o)
PyObject *o;
@ -736,69 +757,42 @@ PyNumber_Int(o)
if (o == NULL)
return null_error();
if (PyInt_Check(o)) {
Py_INCREF(o);
return o;
}
if (PyString_Check(o))
return PyInt_FromString(PyString_AS_STRING(o), NULL, 10);
return int_from_string(PyString_AS_STRING(o),
PyString_GET_SIZE(o));
if (PyUnicode_Check(o))
return PyInt_FromUnicode(PyUnicode_AS_UNICODE(o),
PyUnicode_GET_SIZE(o),
10);
m = o->ob_type->tp_as_number;
if (m && m->nb_int)
return m->nb_int(o);
if (!PyObject_AsCharBuffer(o, &buffer, &buffer_len))
return PyInt_FromString((char*)buffer, NULL, 10);
return int_from_string((char*)buffer, buffer_len);
return type_error("object can't be converted to int");
}
/* There are two C API functions for converting a string to a long,
* PyNumber_Long() and PyLong_FromString(). Both are used in builtin_long,
* reachable from Python with the built-in function long().
*
* The difference is this: PyNumber_Long will raise an exception when the
* string cannot be converted to a long. The most common situation is
* where a float string is passed in; this raises a ValueError.
* PyLong_FromString does not raise an exception; it silently truncates the
* float to an integer.
*
* You can see the different behavior from Python with the following:
*
* long('9.5')
* => ValueError: invalid literal for long(): 9.5
*
* long('9.5', 10)
* => 9L
*
* The first example ends up calling PyNumber_Long(), while the second one
* calls PyLong_FromString().
*/
/* Add a check for embedded NULL-bytes in the argument. */
static PyObject *
long_from_string(s, len)
const char *s;
int len;
{
const char *start;
char *end;
PyObject *x;
char buffer[256]; /* For errors */
start = s;
while (*s && isspace(Py_CHARMASK(*s)))
s++;
x = PyLong_FromString((char*)s, &end, 10);
if (x == NULL) {
if (PyErr_ExceptionMatches(PyExc_ValueError))
goto bad;
if (x == NULL)
return NULL;
}
while (*end && isspace(Py_CHARMASK(*end)))
end++;
if (*end != '\0') {
bad:
sprintf(buffer, "invalid literal for long(): %.200s", s);
PyErr_SetString(PyExc_ValueError, buffer);
Py_XDECREF(x);
return NULL;
}
else if (end != start + len) {
if (end != s + len) {
PyErr_SetString(PyExc_ValueError,
"null byte in argument for long()");
Py_DECREF(x);
return NULL;
}
return x;
@ -814,6 +808,10 @@ PyNumber_Long(o)
if (o == NULL)
return null_error();
if (PyLong_Check(o)) {
Py_INCREF(o);
return o;
}
if (PyString_Check(o))
/* need to do extra error checking that PyLong_FromString()
* doesn't do. In particular long('9.5') must raise an
@ -821,6 +819,11 @@ PyNumber_Long(o)
*/
return long_from_string(PyString_AS_STRING(o),
PyString_GET_SIZE(o));
if (PyUnicode_Check(o))
/* The above check is done in PyLong_FromUnicode(). */
return PyLong_FromUnicode(PyUnicode_AS_UNICODE(o),
PyUnicode_GET_SIZE(o),
10);
m = o->ob_type->tp_as_number;
if (m && m->nb_long)
return m->nb_long(o);
@ -838,6 +841,10 @@ PyNumber_Float(o)
if (o == NULL)
return null_error();
if (PyFloat_Check(o)) {
Py_INCREF(o);
return o;
}
if (!PyString_Check(o)) {
m = o->ob_type->tp_as_number;
if (m && m->nb_float)

16
Objects/floatobject.c
View file

@ -164,6 +164,22 @@ PyFloat_FromString(v, pend)
s = PyString_AS_STRING(v);
len = PyString_GET_SIZE(v);
}
else if (PyUnicode_Check(v)) {
char s_buffer[256];
if (PyUnicode_GET_SIZE(v) >= sizeof(s_buffer)) {
PyErr_SetString(PyExc_ValueError,
"float() literal too large to convert");
return NULL;
}
if (PyUnicode_EncodeDecimal(PyUnicode_AS_UNICODE(v),
PyUnicode_GET_SIZE(v),
s_buffer,
NULL))
return NULL;
s = s_buffer;
len = strlen(s);
}
else if (PyObject_AsCharBuffer(v, &s, &len)) {
PyErr_SetString(PyExc_TypeError,
"float() needs a string argument");

18
Objects/intobject.c
View file

@ -261,6 +261,24 @@ PyInt_FromString(s, pend, base)
return PyInt_FromLong(x);
}
PyObject *
PyInt_FromUnicode(s, length, base)
Py_UNICODE *s;
int length;
int base;
{
char buffer[256];
if (length >= sizeof(buffer)) {
PyErr_SetString(PyExc_ValueError,
"int() literal too large to convert");
return NULL;
}
if (PyUnicode_EncodeDecimal(s, length, buffer, NULL))
return NULL;
return PyInt_FromString(buffer, NULL, base);
}
/* Methods */
/* ARGSUSED */

41
Objects/longobject.c
View file

@ -724,7 +724,7 @@ PyLong_FromString(str, pend, base)
int base;
{
int sign = 1;
char *start;
char *start, *orig_str = str;
PyLongObject *z;
if ((base != 0 && base < 2) || base > 36) {
@ -772,17 +772,44 @@ PyLong_FromString(str, pend, base)
}
if (z == NULL)
return NULL;
if (str == start) {
PyErr_SetString(PyExc_ValueError,
"no digits in long int constant");
Py_DECREF(z);
return NULL;
}
if (str == start)
goto onError;
if (sign < 0 && z != NULL && z->ob_size != 0)
z->ob_size = -(z->ob_size);
if (*str == 'L' || *str == 'l')
str++;
while (*str && isspace(Py_CHARMASK(*str)))
str++;
if (*str != '\0')
goto onError;
if (pend)
*pend = str;
return (PyObject *) z;
onError:
PyErr_Format(PyExc_ValueError,
"invalid literal for long(): %.200s", orig_str);
Py_XDECREF(z);
return NULL;
}
PyObject *
PyLong_FromUnicode(u, length, base)
Py_UNICODE *u;
int length;
int base;
{
char buffer[256];
if (length >= sizeof(buffer)) {
PyErr_SetString(PyExc_ValueError,
"long() literal too large to convert");
return NULL;
}
if (PyUnicode_EncodeDecimal(u, length, buffer, NULL))
return NULL;
return PyLong_FromString(buffer, NULL, base);
}
static PyLongObject *x_divrem

70
Objects/unicodeobject.c
View file

@ -329,8 +329,14 @@ PyObject *PyUnicode_FromObject(register PyObject *obj)
s = PyString_AS_STRING(obj);
len = PyString_GET_SIZE(obj);
}
else if (PyObject_AsCharBuffer(obj, &s, &len))
else if (PyObject_AsCharBuffer(obj, &s, &len)) {
/* Overwrite the error message with something more useful in
case of a TypeError. */
if (PyErr_ExceptionMatches(PyExc_TypeError))
PyErr_SetString(PyExc_TypeError,
"coercing to Unicode: need string or charbuffer");
return NULL;
}
if (len == 0) {
Py_INCREF(unicode_empty);
return (PyObject *)unicode_empty;
@ -1923,6 +1929,60 @@ PyObject *PyUnicode_Translate(PyObject *str,
return NULL;
}
/* --- Decimal Encoder ---------------------------------------------------- */
int PyUnicode_EncodeDecimal(Py_UNICODE *s,
int length,
char *output,
const char *errors)
{
Py_UNICODE *p, *end;
if (output == NULL) {
PyErr_BadArgument();
return -1;
}
p = s;
end = s + length;
while (p < end) {
register Py_UNICODE ch = *p++;
int decimal;
if (Py_UNICODE_ISSPACE(ch)) {
*output++ = ' ';
continue;
}
decimal = Py_UNICODE_TODECIMAL(ch);
if (decimal >= 0) {
*output++ = '0' + decimal;
continue;
}
if (0 < ch < 256) {
*output++ = ch;
continue;
}
/* All other characters are considered invalid */
if (errors == NULL || strcmp(errors, "strict") == 0) {
PyErr_SetString(PyExc_ValueError,
"invalid decimal Unicode string");
goto onError;
}
else if (strcmp(errors, "ignore") == 0)
continue;
else if (strcmp(errors, "replace") == 0) {
*output++ = '?';
continue;
}
}
/* 0-terminate the output string */
*output++ = '\0';
return 0;
onError:
return -1;
}
/* --- Helpers ------------------------------------------------------------ */
static
@ -2811,12 +2871,14 @@ int PyUnicode_Contains(PyObject *container,
register Py_UNICODE ch;
/* Coerce the two arguments */
u = (PyUnicodeObject *)PyUnicode_FromObject(container);
if (u == NULL)
goto onError;
v = (PyUnicodeObject *)PyUnicode_FromObject(element);
if (v == NULL)
goto onError;
u = (PyUnicodeObject *)PyUnicode_FromObject(container);
if (u == NULL) {
Py_DECREF(v);
goto onError;
}
/* Check v in u */
if (PyUnicode_GET_SIZE(v) != 1) {

53
Python/bltinmodule.c
View file

@ -449,17 +449,44 @@ complex_from_string(v)
PyObject *v;
{
extern double strtod Py_PROTO((const char *, char **));
char *s, *start, *end;
const char *s, *start;
char *end;
double x=0.0, y=0.0, z;
int got_re=0, got_im=0, done=0;
int digit_or_dot;
int sw_error=0;
int sign;
char buffer[256]; /* For errors */
int len;
start = s = PyString_AS_STRING(v);
if (PyString_Check(v)) {
s = PyString_AS_STRING(v);
len = PyString_GET_SIZE(v);
}
else if (PyUnicode_Check(v)) {
char s_buffer[256];
if (PyUnicode_GET_SIZE(v) >= sizeof(s_buffer)) {
PyErr_SetString(PyExc_ValueError,
"complex() literal too large to convert");
return NULL;
}
if (PyUnicode_EncodeDecimal(PyUnicode_AS_UNICODE(v),
PyUnicode_GET_SIZE(v),
s_buffer,
NULL))
return NULL;
s = s_buffer;
len = strlen(s);
}
else if (PyObject_AsCharBuffer(v, &s, &len)) {
PyErr_SetString(PyExc_TypeError,
"complex() needs a string first argument");
return NULL;
}
/* position on first nonblank */
start = s;
while (*s && isspace(Py_CHARMASK(*s)))
s++;
if (s[0] == '\0') {
@ -475,7 +502,7 @@ complex_from_string(v)
switch (*s) {
case '\0':
if (s-start != PyString_GET_SIZE(v)) {
if (s-start != len) {
PyErr_SetString(
PyExc_ValueError,
"null byte in argument for complex()");
@ -584,7 +611,7 @@ builtin_complex(self, args)
i = NULL;
if (!PyArg_ParseTuple(args, "O|O:complex", &r, &i))
return NULL;
if (PyString_Check(r))
if (PyString_Check(r) || PyUnicode_Check(r))
return complex_from_string(r);
if ((nbr = r->ob_type->tp_as_number) == NULL ||
nbr->nb_float == NULL ||
@ -1289,12 +1316,17 @@ builtin_int(self, args)
return NULL;
if (base == -909)
return PyNumber_Int(v);
else if (!PyString_Check(v)) {
else if (PyString_Check(v))
return PyInt_FromString(PyString_AS_STRING(v), NULL, base);
else if (PyUnicode_Check(v))
return PyInt_FromUnicode(PyUnicode_AS_UNICODE(v),
PyUnicode_GET_SIZE(v),
base);
else {
PyErr_SetString(PyExc_TypeError,
"can't convert non-string with explicit base");
return NULL;
}
return PyInt_FromString(PyString_AS_STRING(v), NULL, base);
}
static char int_doc[] =
@ -1319,12 +1351,17 @@ builtin_long(self, args)
return NULL;
if (base == -909)
return PyNumber_Long(v);
else if (!PyString_Check(v)) {
else if (PyString_Check(v))
return PyLong_FromString(PyString_AS_STRING(v), NULL, base);
else if (PyUnicode_Check(v))
return PyLong_FromUnicode(PyUnicode_AS_UNICODE(v),
PyUnicode_GET_SIZE(v),
base);
else {
PyErr_SetString(PyExc_TypeError,
"can't convert non-string with explicit base");
return NULL;
}
return PyLong_FromString(PyString_AS_STRING(v), NULL, base);
}
static char long_doc[] =

21
Python/codecs.c
View file

@ -84,8 +84,11 @@ int PyCodec_Register(PyObject *search_function)
return -1;
}
/* Convert a string to a normalized Python string: all characters are
converted to lower case, spaces are replaced with underscores. */
static
PyObject *lowercasestring(const char *string)
PyObject *normalizestring(const char *string)
{
register int i;
int len = strlen(string);
@ -96,8 +99,14 @@ PyObject *lowercasestring(const char *string)
if (v == NULL)
return NULL;
p = PyString_AS_STRING(v);
for (i = 0; i < len; i++)
p[i] = tolower(string[i]);
for (i = 0; i < len; i++) {
register char ch = string[i];
if (ch == ' ')
ch = '-';
else
ch = tolower(ch);
p[i] = ch;
}
return v;
}
@ -132,8 +141,10 @@ PyObject *_PyCodec_Lookup(const char *encoding)
goto onError;
}
/* Convert the encoding to a lower-cased Python string */
v = lowercasestring(encoding);
/* Convert the encoding to a normalized Python string: all
characters are converted to lower case, spaces and hypens are
replaced with underscores. */
v = normalizestring(encoding);
if (v == NULL)
goto onError;
PyString_InternInPlace(&v);