mirror of
https://github.com/python/cpython
synced 2024-09-05 16:38:56 +00:00
Convert code from sys.stdin.encoding to UTF-8 in
interactive mode. Fixes #1100.
This commit is contained in:
parent
53de1902e7
commit
85bcc66bb4
|
@ -34,7 +34,8 @@ PyAPI_FUNC(node *) PyParser_ParseFile (FILE *, const char *, grammar *, int,
|
|||
|
||||
PyAPI_FUNC(node *) PyParser_ParseStringFlags(const char *, grammar *, int,
|
||||
perrdetail *, int);
|
||||
PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *, grammar *,
|
||||
PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *,
|
||||
const char*, grammar *,
|
||||
int, char *, char *,
|
||||
perrdetail *, int);
|
||||
|
||||
|
|
|
@ -40,7 +40,8 @@ PyAPI_FUNC(int) PyRun_InteractiveLoopFlags(FILE *, const char *, PyCompilerFlags
|
|||
PyAPI_FUNC(struct _mod *) PyParser_ASTFromString(const char *, const char *,
|
||||
int, PyCompilerFlags *flags,
|
||||
PyArena *);
|
||||
PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(FILE *, const char *, int,
|
||||
PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(FILE *, const char *,
|
||||
const char*, int,
|
||||
char *, char *,
|
||||
PyCompilerFlags *, int *,
|
||||
PyArena *);
|
||||
|
|
|
@ -59,19 +59,20 @@ node *
|
|||
PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,
|
||||
char *ps1, char *ps2, perrdetail *err_ret)
|
||||
{
|
||||
return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2,
|
||||
err_ret, 0);
|
||||
return PyParser_ParseFileFlags(fp, filename, NULL,
|
||||
g, start, ps1, ps2, err_ret, 0);
|
||||
}
|
||||
|
||||
node *
|
||||
PyParser_ParseFileFlags(FILE *fp, const char *filename, grammar *g, int start,
|
||||
PyParser_ParseFileFlags(FILE *fp, const char *filename, const char* enc,
|
||||
grammar *g, int start,
|
||||
char *ps1, char *ps2, perrdetail *err_ret, int flags)
|
||||
{
|
||||
struct tok_state *tok;
|
||||
|
||||
initerr(err_ret, filename);
|
||||
|
||||
if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) {
|
||||
if ((tok = PyTokenizer_FromFile(fp, enc, ps1, ps2)) == NULL) {
|
||||
err_ret->error = E_NOMEM;
|
||||
return NULL;
|
||||
}
|
||||
|
|
|
@ -677,7 +677,7 @@ PyTokenizer_FromString(const char *str)
|
|||
/* Set up tokenizer for file */
|
||||
|
||||
struct tok_state *
|
||||
PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2)
|
||||
PyTokenizer_FromFile(FILE *fp, char* enc, char *ps1, char *ps2)
|
||||
{
|
||||
struct tok_state *tok = tok_new();
|
||||
if (tok == NULL)
|
||||
|
@ -691,6 +691,17 @@ PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2)
|
|||
tok->fp = fp;
|
||||
tok->prompt = ps1;
|
||||
tok->nextprompt = ps2;
|
||||
if (enc != NULL) {
|
||||
/* Must copy encoding declaration since it
|
||||
gets copied into the parse tree. */
|
||||
tok->encoding = PyMem_MALLOC(strlen(enc)+1);
|
||||
if (!tok->encoding) {
|
||||
PyTokenizer_Free(tok);
|
||||
return NULL;
|
||||
}
|
||||
strcpy(tok->encoding, enc);
|
||||
tok->decoding_state = -1;
|
||||
}
|
||||
return tok;
|
||||
}
|
||||
|
||||
|
@ -742,6 +753,29 @@ tok_nextc(register struct tok_state *tok)
|
|||
}
|
||||
if (tok->prompt != NULL) {
|
||||
char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);
|
||||
#ifndef PGEN
|
||||
if (tok->encoding && newtok && *newtok) {
|
||||
/* Recode to UTF-8 */
|
||||
Py_ssize_t buflen;
|
||||
const char* buf;
|
||||
PyObject *u = translate_into_utf8(newtok, tok->encoding);
|
||||
PyMem_FREE(newtok);
|
||||
if (!u) {
|
||||
tok->done = E_DECODE;
|
||||
return EOF;
|
||||
}
|
||||
buflen = PyBytes_Size(u);
|
||||
buf = PyBytes_AsString(u);
|
||||
if (!buf) {
|
||||
Py_DECREF(u);
|
||||
tok->done = E_DECODE;
|
||||
return EOF;
|
||||
}
|
||||
newtok = PyMem_MALLOC(buflen+1);
|
||||
strcpy(newtok, buf);
|
||||
Py_DECREF(u);
|
||||
}
|
||||
#endif
|
||||
if (tok->nextprompt != NULL)
|
||||
tok->prompt = tok->nextprompt;
|
||||
if (newtok == NULL)
|
||||
|
|
|
@ -55,7 +55,8 @@ struct tok_state {
|
|||
};
|
||||
|
||||
extern struct tok_state *PyTokenizer_FromString(const char *);
|
||||
extern struct tok_state *PyTokenizer_FromFile(FILE *, char *, char *);
|
||||
extern struct tok_state *PyTokenizer_FromFile(FILE *, char*,
|
||||
char *, char *);
|
||||
extern void PyTokenizer_Free(struct tok_state *);
|
||||
extern int PyTokenizer_Get(struct tok_state *, char **, char **);
|
||||
|
||||
|
|
|
@ -809,7 +809,8 @@ parse_source_module(const char *pathname, FILE *fp)
|
|||
if (arena == NULL)
|
||||
return NULL;
|
||||
|
||||
mod = PyParser_ASTFromFile(fp, pathname, Py_file_input, 0, 0, 0,
|
||||
mod = PyParser_ASTFromFile(fp, pathname, NULL,
|
||||
Py_file_input, 0, 0, 0,
|
||||
NULL, arena);
|
||||
if (mod) {
|
||||
co = PyAST_Compile(mod, pathname, NULL, arena);
|
||||
|
|
|
@ -744,12 +744,22 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flag
|
|||
int
|
||||
PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags)
|
||||
{
|
||||
PyObject *m, *d, *v, *w;
|
||||
PyObject *m, *d, *v, *w, *oenc = NULL;
|
||||
mod_ty mod;
|
||||
PyArena *arena;
|
||||
char *ps1 = "", *ps2 = "";
|
||||
char *ps1 = "", *ps2 = "", *enc = NULL;
|
||||
int errcode = 0;
|
||||
|
||||
if (fp == stdin) {
|
||||
/* Fetch encoding from sys.stdin */
|
||||
v = PySys_GetObject("stdin");
|
||||
if (!v)
|
||||
return -1;
|
||||
oenc = PyObject_GetAttrString(v, "encoding");
|
||||
if (!oenc)
|
||||
return -1;
|
||||
enc = PyUnicode_AsString(oenc);
|
||||
}
|
||||
v = PySys_GetObject("ps1");
|
||||
if (v != NULL) {
|
||||
v = PyObject_Str(v);
|
||||
|
@ -770,13 +780,15 @@ PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags
|
|||
if (arena == NULL) {
|
||||
Py_XDECREF(v);
|
||||
Py_XDECREF(w);
|
||||
Py_XDECREF(oenc);
|
||||
return -1;
|
||||
}
|
||||
mod = PyParser_ASTFromFile(fp, filename,
|
||||
mod = PyParser_ASTFromFile(fp, filename, enc,
|
||||
Py_single_input, ps1, ps2,
|
||||
flags, &errcode, arena);
|
||||
Py_XDECREF(v);
|
||||
Py_XDECREF(w);
|
||||
Py_XDECREF(oenc);
|
||||
if (mod == NULL) {
|
||||
PyArena_Free(arena);
|
||||
if (errcode == E_EOF) {
|
||||
|
@ -1254,7 +1266,7 @@ PyRun_FileExFlags(FILE *fp, const char *filename, int start, PyObject *globals,
|
|||
if (arena == NULL)
|
||||
return NULL;
|
||||
|
||||
mod = PyParser_ASTFromFile(fp, filename, start, 0, 0,
|
||||
mod = PyParser_ASTFromFile(fp, filename, NULL, start, 0, 0,
|
||||
flags, NULL, arena);
|
||||
if (closeit)
|
||||
fclose(fp);
|
||||
|
@ -1379,13 +1391,15 @@ PyParser_ASTFromString(const char *s, const char *filename, int start,
|
|||
}
|
||||
|
||||
mod_ty
|
||||
PyParser_ASTFromFile(FILE *fp, const char *filename, int start, char *ps1,
|
||||
PyParser_ASTFromFile(FILE *fp, const char *filename, const char* enc,
|
||||
int start, char *ps1,
|
||||
char *ps2, PyCompilerFlags *flags, int *errcode,
|
||||
PyArena *arena)
|
||||
{
|
||||
mod_ty mod;
|
||||
perrdetail err;
|
||||
node *n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar,
|
||||
node *n = PyParser_ParseFileFlags(fp, filename, enc,
|
||||
&_PyParser_Grammar,
|
||||
start, ps1, ps2, &err, PARSER_FLAGS(flags));
|
||||
if (n) {
|
||||
mod = PyAST_FromNode(n, flags, filename, arena);
|
||||
|
@ -1406,7 +1420,8 @@ node *
|
|||
PyParser_SimpleParseFileFlags(FILE *fp, const char *filename, int start, int flags)
|
||||
{
|
||||
perrdetail err;
|
||||
node *n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar,
|
||||
node *n = PyParser_ParseFileFlags(fp, filename, NULL,
|
||||
&_PyParser_Grammar,
|
||||
start, NULL, NULL, &err, flags);
|
||||
if (n == NULL)
|
||||
err_input(&err);
|
||||
|
|
Loading…
Reference in a new issue