mirror of
https://github.com/python/cpython
synced 2024-09-05 16:38:56 +00:00
Merged revisions 61952-61953 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/trunk ........ r61952 | mark.dickinson | 2008-03-26 22:41:36 +0100 (Wed, 26 Mar 2008) | 2 lines Typo: "objects reference count" -> "object's reference count" ........ r61953 | christian.heimes | 2008-03-26 23:01:37 +0100 (Wed, 26 Mar 2008) | 4 lines Patch #2477: Added from __future__ import unicode_literals The new PyParser_*Ex() functions are based on Neal's suggestion and initial patch. The new __future__ feature makes all '' and r'' unicode strings. b'' and br'' stay (byte) strings. ........
This commit is contained in:
parent
9edef04c95
commit
4d6ec85a02
|
@ -20,7 +20,7 @@ definition of all other Python objects.
|
|||
|
||||
All object types are extensions of this type. This is a type which contains the
|
||||
information Python needs to treat a pointer to an object as an object. In a
|
||||
normal "release" build, it contains only the objects reference count and a
|
||||
normal "release" build, it contains only the object's reference count and a
|
||||
pointer to the corresponding type object. It corresponds to the fields defined
|
||||
by the expansion of the ``PyObject_HEAD`` macro.
|
||||
|
||||
|
|
|
@ -49,6 +49,7 @@ typedef struct {
|
|||
#define CO_FUTURE_ABSOLUTE_IMPORT 0x4000 /* do absolute imports by default */
|
||||
#define CO_FUTURE_WITH_STATEMENT 0x8000
|
||||
#define CO_FUTURE_PRINT_FUNCTION 0x10000
|
||||
#define CO_FUTURE_UNICODE_LITERALS 0x20000
|
||||
#endif
|
||||
|
||||
/* This should be defined if a future statement modifies the syntax.
|
||||
|
|
|
@ -25,6 +25,7 @@ typedef struct {
|
|||
#define FUTURE_ABSOLUTE_IMPORT "absolute_import"
|
||||
#define FUTURE_WITH_STATEMENT "with_statement"
|
||||
#define FUTURE_PRINT_FUNCTION "print_function"
|
||||
#define FUTURE_UNICODE_LITERALS "unicode_literals"
|
||||
|
||||
struct _mod; /* Declare the existence of this type */
|
||||
PyAPI_FUNC(PyCodeObject *) PyAST_Compile(struct _mod *, const char *,
|
||||
|
|
|
@ -25,6 +25,8 @@ typedef struct {
|
|||
|
||||
#if 0
|
||||
#define PyPARSE_WITH_IS_KEYWORD 0x0003
|
||||
#define PyPARSE_PRINT_IS_FUNCTION 0x0004
|
||||
#define PyPARSE_UNICODE_LITERALS 0x0008
|
||||
#endif
|
||||
|
||||
PyAPI_FUNC(node *) PyParser_ParseString(const char *, grammar *, int,
|
||||
|
@ -38,11 +40,19 @@ PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *,
|
|||
const char*, grammar *,
|
||||
int, char *, char *,
|
||||
perrdetail *, int);
|
||||
PyAPI_FUNC(node *) PyParser_ParseFileFlagsEx(FILE *, const char *,
|
||||
const char*, grammar *,
|
||||
int, char *, char *,
|
||||
perrdetail *, int *);
|
||||
|
||||
PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilename(const char *,
|
||||
const char *,
|
||||
grammar *, int,
|
||||
perrdetail *, int);
|
||||
PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilenameEx(const char *,
|
||||
const char *,
|
||||
grammar *, int,
|
||||
perrdetail *, int *);
|
||||
|
||||
/* Note that he following function is defined in pythonrun.c not parsetok.c. */
|
||||
PyAPI_FUNC(void) PyParser_SetError(perrdetail *);
|
||||
|
|
|
@ -54,6 +54,7 @@
|
|||
"absolute_import",
|
||||
"with_statement",
|
||||
"print_function",
|
||||
"unicode_literals",
|
||||
]
|
||||
|
||||
__all__ = ["all_feature_names"] + all_feature_names
|
||||
|
@ -68,6 +69,7 @@
|
|||
CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 # perform absolute imports by default
|
||||
CO_FUTURE_WITH_STATEMENT = 0x8000 # with statement
|
||||
CO_FUTURE_PRINT_FUNCTION = 0x10000 # print function
|
||||
CO_FUTURE_UNICODE_LITERALS = 0x20000 # unicode string literals
|
||||
|
||||
class _Feature:
|
||||
def __init__(self, optionalRelease, mandatoryRelease, compiler_flag):
|
||||
|
@ -120,3 +122,7 @@ def __repr__(self):
|
|||
print_function = _Feature((2, 6, 0, "alpha", 2),
|
||||
(3, 0, 0, "alpha", 0),
|
||||
CO_FUTURE_PRINT_FUNCTION)
|
||||
|
||||
unicode_literals = _Feature((2, 6, 0, "alpha", 2),
|
||||
(3, 0, 0, "alpha", 0),
|
||||
CO_FUTURE_UNICODE_LITERALS)
|
||||
|
|
|
@ -206,10 +206,18 @@ future_hack(parser_state *ps)
|
|||
|
||||
for (i = 0; i < NCH(ch); i += 2) {
|
||||
cch = CHILD(ch, i);
|
||||
if (NCH(cch) >= 1 && TYPE(CHILD(cch, 0)) == NAME &&
|
||||
strcmp(STR(CHILD(cch, 0)), "with_statement") == 0) {
|
||||
if (NCH(cch) >= 1 && TYPE(CHILD(cch, 0)) == NAME) {
|
||||
char *str_ch = STR(CHILD(cch, 0));
|
||||
if (strcmp(str_ch, FUTURE_WITH_STATEMENT) == 0) {
|
||||
ps->p_flags |= CO_FUTURE_WITH_STATEMENT;
|
||||
break;
|
||||
} else if (strcmp(str_ch, FUTURE_PRINT_FUNCTION) == 0) {
|
||||
ps->p_flags |= CO_FUTURE_PRINT_FUNCTION;
|
||||
break;
|
||||
} else if (strcmp(str_ch, FUTURE_UNICODE_LITERALS) == 0) {
|
||||
ps->p_flags |= CO_FUTURE_UNICODE_LITERALS;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -14,7 +14,7 @@ int Py_TabcheckFlag;
|
|||
|
||||
|
||||
/* Forward */
|
||||
static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int);
|
||||
static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *);
|
||||
static void initerr(perrdetail *err_ret, const char* filename);
|
||||
|
||||
/* Parse input coming from a string. Return error code, print some errors. */
|
||||
|
@ -36,6 +36,16 @@ node *
|
|||
PyParser_ParseStringFlagsFilename(const char *s, const char *filename,
|
||||
grammar *g, int start,
|
||||
perrdetail *err_ret, int flags)
|
||||
{
|
||||
int iflags = flags;
|
||||
return PyParser_ParseStringFlagsFilenameEx(s, filename, g, start,
|
||||
err_ret, &iflags);
|
||||
}
|
||||
|
||||
node *
|
||||
PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename,
|
||||
grammar *g, int start,
|
||||
perrdetail *err_ret, int *flags)
|
||||
{
|
||||
struct tok_state *tok;
|
||||
|
||||
|
@ -64,9 +74,19 @@ PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,
|
|||
}
|
||||
|
||||
node *
|
||||
PyParser_ParseFileFlags(FILE *fp, const char *filename, const char* enc,
|
||||
PyParser_ParseFileFlags(FILE *fp, const char *filename, const char *enc,
|
||||
grammar *g, int start,
|
||||
char *ps1, char *ps2, perrdetail *err_ret, int flags)
|
||||
{
|
||||
int iflags = flags;
|
||||
return PyParser_ParseFileFlagsEx(fp, filename, enc, g, start, ps1,
|
||||
ps2, err_ret, &iflags);
|
||||
}
|
||||
|
||||
node *
|
||||
PyParser_ParseFileFlagsEx(FILE *fp, const char *filename,
|
||||
const char *enc, grammar *g, int start,
|
||||
char *ps1, char *ps2, perrdetail *err_ret, int *flags)
|
||||
{
|
||||
struct tok_state *tok;
|
||||
|
||||
|
@ -104,7 +124,7 @@ warn(const char *msg, const char *filename, int lineno)
|
|||
|
||||
static node *
|
||||
parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
|
||||
int flags)
|
||||
int *flags)
|
||||
{
|
||||
parser_state *ps;
|
||||
node *n;
|
||||
|
@ -117,7 +137,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
|
|||
return NULL;
|
||||
}
|
||||
#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
|
||||
if (flags & PyPARSE_WITH_IS_KEYWORD)
|
||||
if (*flags & PyPARSE_WITH_IS_KEYWORD)
|
||||
ps->p_flags |= CO_FUTURE_WITH_STATEMENT;
|
||||
#endif
|
||||
|
||||
|
@ -141,7 +161,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
|
|||
except if a certain flag is given --
|
||||
codeop.py uses this. */
|
||||
if (tok->indent &&
|
||||
!(flags & PyPARSE_DONT_IMPLY_DEDENT))
|
||||
!(*flags & PyPARSE_DONT_IMPLY_DEDENT))
|
||||
{
|
||||
tok->pendin = -tok->indent;
|
||||
tok->indent = 0;
|
||||
|
@ -205,7 +225,9 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
|
|||
}
|
||||
else
|
||||
n = NULL;
|
||||
|
||||
#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
|
||||
*flags = ps->p_flags;
|
||||
#endif
|
||||
PyParser_Delete(ps);
|
||||
|
||||
if (n == NULL) {
|
||||
|
|
24
Python/ast.c
24
Python/ast.c
|
@ -35,7 +35,7 @@ static stmt_ty ast_for_classdef(struct compiling *, const node *, asdl_seq *);
|
|||
static expr_ty ast_for_call(struct compiling *, const node *, expr_ty);
|
||||
|
||||
static PyObject *parsenumber(const char *);
|
||||
static PyObject *parsestr(const node *n, const char *encoding, int *bytesmode);
|
||||
static PyObject *parsestr(struct compiling *, const node *n, int *bytesmode);
|
||||
static PyObject *parsestrplus(struct compiling *, const node *n,
|
||||
int *bytesmode);
|
||||
|
||||
|
@ -3191,14 +3191,13 @@ decode_unicode(const char *s, size_t len, int rawmode, const char *encoding)
|
|||
* parsestr parses it, and returns the decoded Python string object.
|
||||
*/
|
||||
static PyObject *
|
||||
parsestr(const node *n, const char *encoding, int *bytesmode)
|
||||
parsestr(struct compiling *c, const node *n, int *bytesmode)
|
||||
{
|
||||
size_t len;
|
||||
const char *s = STR(n);
|
||||
int quote = Py_CHARMASK(*s);
|
||||
int rawmode = 0;
|
||||
int need_encoding;
|
||||
|
||||
if (isalpha(quote)) {
|
||||
if (quote == 'b' || quote == 'B') {
|
||||
quote = *++s;
|
||||
|
@ -3233,7 +3232,7 @@ parsestr(const node *n, const char *encoding, int *bytesmode)
|
|||
}
|
||||
}
|
||||
if (!*bytesmode && !rawmode) {
|
||||
return decode_unicode(s, len, rawmode, encoding);
|
||||
return decode_unicode(s, len, rawmode, c->c_encoding);
|
||||
}
|
||||
if (*bytesmode) {
|
||||
/* Disallow non-ascii characters (but not escapes) */
|
||||
|
@ -3246,28 +3245,27 @@ parsestr(const node *n, const char *encoding, int *bytesmode)
|
|||
}
|
||||
}
|
||||
}
|
||||
need_encoding = (!*bytesmode && encoding != NULL &&
|
||||
strcmp(encoding, "utf-8") != 0 &&
|
||||
strcmp(encoding, "iso-8859-1") != 0);
|
||||
need_encoding = (!*bytesmode && c->c_encoding != NULL &&
|
||||
strcmp(c->c_encoding, "utf-8") != 0 &&
|
||||
strcmp(c->c_encoding, "iso-8859-1") != 0);
|
||||
if (rawmode || strchr(s, '\\') == NULL) {
|
||||
if (need_encoding) {
|
||||
PyObject *v, *u = PyUnicode_DecodeUTF8(s, len, NULL);
|
||||
if (u == NULL || !*bytesmode)
|
||||
return u;
|
||||
v = PyUnicode_AsEncodedString(u, encoding, NULL);
|
||||
v = PyUnicode_AsEncodedString(u, c->c_encoding, NULL);
|
||||
Py_DECREF(u);
|
||||
return v;
|
||||
} else if (*bytesmode) {
|
||||
return PyString_FromStringAndSize(s, len);
|
||||
} else if (strcmp(encoding, "utf-8") == 0) {
|
||||
} else if (strcmp(c->c_encoding, "utf-8") == 0) {
|
||||
return PyUnicode_FromStringAndSize(s, len);
|
||||
} else {
|
||||
return PyUnicode_DecodeLatin1(s, len, NULL);
|
||||
}
|
||||
}
|
||||
|
||||
return PyString_DecodeEscape(s, len, NULL, 1,
|
||||
need_encoding ? encoding : NULL);
|
||||
need_encoding ? c->c_encoding : NULL);
|
||||
}
|
||||
|
||||
/* Build a Python string object out of a STRING+ atom. This takes care of
|
||||
|
@ -3280,13 +3278,13 @@ parsestrplus(struct compiling *c, const node *n, int *bytesmode)
|
|||
PyObject *v;
|
||||
int i;
|
||||
REQ(CHILD(n, 0), STRING);
|
||||
v = parsestr(CHILD(n, 0), c->c_encoding, bytesmode);
|
||||
v = parsestr(c, CHILD(n, 0), bytesmode);
|
||||
if (v != NULL) {
|
||||
/* String literal concatenation */
|
||||
for (i = 1; i < NCH(n); i++) {
|
||||
PyObject *s;
|
||||
int subbm = 0;
|
||||
s = parsestr(CHILD(n, i), c->c_encoding, &subbm);
|
||||
s = parsestr(c, CHILD(n, i), &subbm);
|
||||
if (s == NULL)
|
||||
goto onError;
|
||||
if (*bytesmode != subbm) {
|
||||
|
|
|
@ -35,6 +35,8 @@ future_check_features(PyFutureFeatures *ff, stmt_ty s, const char *filename)
|
|||
continue;
|
||||
} else if (strcmp(feature, FUTURE_PRINT_FUNCTION) == 0) {
|
||||
continue;
|
||||
} else if (strcmp(feature, FUTURE_UNICODE_LITERALS) == 0) {
|
||||
continue;
|
||||
} else if (strcmp(feature, "braces") == 0) {
|
||||
PyErr_SetString(PyExc_SyntaxError,
|
||||
"not a chance");
|
||||
|
|
|
@ -821,12 +821,13 @@ parse_source_module(const char *pathname, FILE *fp)
|
|||
{
|
||||
PyCodeObject *co = NULL;
|
||||
mod_ty mod;
|
||||
PyCompilerFlags flags;
|
||||
PyArena *arena = PyArena_New();
|
||||
if (arena == NULL)
|
||||
return NULL;
|
||||
|
||||
mod = PyParser_ASTFromFile(fp, pathname, NULL,
|
||||
Py_file_input, 0, 0, 0,
|
||||
Py_file_input, 0, 0, &flags,
|
||||
NULL, arena);
|
||||
if (mod) {
|
||||
co = PyAST_Compile(mod, pathname, NULL, arena);
|
||||
|
|
|
@ -1563,11 +1563,12 @@ Py_SymtableString(const char *str, const char *filename, int start)
|
|||
{
|
||||
struct symtable *st;
|
||||
mod_ty mod;
|
||||
PyCompilerFlags flags;
|
||||
PyArena *arena = PyArena_New();
|
||||
if (arena == NULL)
|
||||
return NULL;
|
||||
|
||||
mod = PyParser_ASTFromString(str, filename, start, NULL, arena);
|
||||
mod = PyParser_ASTFromString(str, filename, start, &flags, arena);
|
||||
if (mod == NULL) {
|
||||
PyArena_Free(arena);
|
||||
return NULL;
|
||||
|
@ -1584,10 +1585,16 @@ PyParser_ASTFromString(const char *s, const char *filename, int start,
|
|||
{
|
||||
mod_ty mod;
|
||||
perrdetail err;
|
||||
node *n = PyParser_ParseStringFlagsFilename(s, filename,
|
||||
int iflags;
|
||||
iflags = PARSER_FLAGS(flags);
|
||||
|
||||
node *n = PyParser_ParseStringFlagsFilenameEx(s, filename,
|
||||
&_PyParser_Grammar, start, &err,
|
||||
PARSER_FLAGS(flags));
|
||||
&iflags);
|
||||
if (n) {
|
||||
if (flags) {
|
||||
flags->cf_flags |= iflags & PyCF_MASK;
|
||||
}
|
||||
mod = PyAST_FromNode(n, flags, filename, arena);
|
||||
PyNode_Free(n);
|
||||
return mod;
|
||||
|
@ -1606,10 +1613,16 @@ PyParser_ASTFromFile(FILE *fp, const char *filename, const char* enc,
|
|||
{
|
||||
mod_ty mod;
|
||||
perrdetail err;
|
||||
node *n = PyParser_ParseFileFlags(fp, filename, enc,
|
||||
int iflags;
|
||||
|
||||
iflags = PARSER_FLAGS(flags);
|
||||
node *n = PyParser_ParseFileFlagsEx(fp, filename, enc,
|
||||
&_PyParser_Grammar,
|
||||
start, ps1, ps2, &err, PARSER_FLAGS(flags));
|
||||
start, ps1, ps2, &err, &iflags);
|
||||
if (n) {
|
||||
if (flags) {
|
||||
flags->cf_flags |= iflags & PyCF_MASK;
|
||||
}
|
||||
mod = PyAST_FromNode(n, flags, filename, arena);
|
||||
PyNode_Free(n);
|
||||
return mod;
|
||||
|
|
Loading…
Reference in a new issue