mirror of
https://github.com/python/cpython
synced 2024-09-16 03:20:00 +00:00
a77aac4fca
To improve the user experience understanding what part of the error messages associated with SyntaxErrors is wrong, we can highlight the whole error range and not only place the caret at the first character. In this way:
>>> foo(x, z for z in range(10), t, w)
File "<stdin>", line 1
foo(x, z for z in range(10), t, w)
^
SyntaxError: Generator expression must be parenthesized
becomes
>>> foo(x, z for z in range(10), t, w)
File "<stdin>", line 1
foo(x, z for z in range(10), t, w)
^^^^^^^^^^^^^^^^^^^^
SyntaxError: Generator expression must be parenthesized
2464 lines
68 KiB
C
2464 lines
68 KiB
C
#include <Python.h>
|
|
#include "pycore_ast.h" // _PyAST_Validate()
|
|
#include <errcode.h>
|
|
#include "tokenizer.h"
|
|
|
|
#include "pegen.h"
|
|
#include "string_parser.h"
|
|
|
|
PyObject *
|
|
_PyPegen_new_type_comment(Parser *p, char *s)
|
|
{
|
|
PyObject *res = PyUnicode_DecodeUTF8(s, strlen(s), NULL);
|
|
if (res == NULL) {
|
|
return NULL;
|
|
}
|
|
if (_PyArena_AddPyObject(p->arena, res) < 0) {
|
|
Py_DECREF(res);
|
|
return NULL;
|
|
}
|
|
return res;
|
|
}
|
|
|
|
arg_ty
|
|
_PyPegen_add_type_comment_to_arg(Parser *p, arg_ty a, Token *tc)
|
|
{
|
|
if (tc == NULL) {
|
|
return a;
|
|
}
|
|
char *bytes = PyBytes_AsString(tc->bytes);
|
|
if (bytes == NULL) {
|
|
return NULL;
|
|
}
|
|
PyObject *tco = _PyPegen_new_type_comment(p, bytes);
|
|
if (tco == NULL) {
|
|
return NULL;
|
|
}
|
|
return _PyAST_arg(a->arg, a->annotation, tco,
|
|
a->lineno, a->col_offset, a->end_lineno, a->end_col_offset,
|
|
p->arena);
|
|
}
|
|
|
|
static int
|
|
init_normalization(Parser *p)
|
|
{
|
|
if (p->normalize) {
|
|
return 1;
|
|
}
|
|
PyObject *m = PyImport_ImportModuleNoBlock("unicodedata");
|
|
if (!m)
|
|
{
|
|
return 0;
|
|
}
|
|
p->normalize = PyObject_GetAttrString(m, "normalize");
|
|
Py_DECREF(m);
|
|
if (!p->normalize)
|
|
{
|
|
return 0;
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
/* Checks if the NOTEQUAL token is valid given the current parser flags
|
|
0 indicates success and nonzero indicates failure (an exception may be set) */
|
|
int
|
|
_PyPegen_check_barry_as_flufl(Parser *p, Token* t) {
|
|
assert(t->bytes != NULL);
|
|
assert(t->type == NOTEQUAL);
|
|
|
|
char* tok_str = PyBytes_AS_STRING(t->bytes);
|
|
if (p->flags & PyPARSE_BARRY_AS_BDFL && strcmp(tok_str, "<>") != 0) {
|
|
RAISE_SYNTAX_ERROR("with Barry as BDFL, use '<>' instead of '!='");
|
|
return -1;
|
|
}
|
|
if (!(p->flags & PyPARSE_BARRY_AS_BDFL)) {
|
|
return strcmp(tok_str, "!=");
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
PyObject *
|
|
_PyPegen_new_identifier(Parser *p, char *n)
|
|
{
|
|
PyObject *id = PyUnicode_DecodeUTF8(n, strlen(n), NULL);
|
|
if (!id) {
|
|
goto error;
|
|
}
|
|
/* PyUnicode_DecodeUTF8 should always return a ready string. */
|
|
assert(PyUnicode_IS_READY(id));
|
|
/* Check whether there are non-ASCII characters in the
|
|
identifier; if so, normalize to NFKC. */
|
|
if (!PyUnicode_IS_ASCII(id))
|
|
{
|
|
PyObject *id2;
|
|
if (!init_normalization(p))
|
|
{
|
|
Py_DECREF(id);
|
|
goto error;
|
|
}
|
|
PyObject *form = PyUnicode_InternFromString("NFKC");
|
|
if (form == NULL)
|
|
{
|
|
Py_DECREF(id);
|
|
goto error;
|
|
}
|
|
PyObject *args[2] = {form, id};
|
|
id2 = _PyObject_FastCall(p->normalize, args, 2);
|
|
Py_DECREF(id);
|
|
Py_DECREF(form);
|
|
if (!id2) {
|
|
goto error;
|
|
}
|
|
if (!PyUnicode_Check(id2))
|
|
{
|
|
PyErr_Format(PyExc_TypeError,
|
|
"unicodedata.normalize() must return a string, not "
|
|
"%.200s",
|
|
_PyType_Name(Py_TYPE(id2)));
|
|
Py_DECREF(id2);
|
|
goto error;
|
|
}
|
|
id = id2;
|
|
}
|
|
PyUnicode_InternInPlace(&id);
|
|
if (_PyArena_AddPyObject(p->arena, id) < 0)
|
|
{
|
|
Py_DECREF(id);
|
|
goto error;
|
|
}
|
|
return id;
|
|
|
|
error:
|
|
p->error_indicator = 1;
|
|
return NULL;
|
|
}
|
|
|
|
static PyObject *
|
|
_create_dummy_identifier(Parser *p)
|
|
{
|
|
return _PyPegen_new_identifier(p, "");
|
|
}
|
|
|
|
static inline Py_ssize_t
|
|
byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset)
|
|
{
|
|
const char *str = PyUnicode_AsUTF8(line);
|
|
if (!str) {
|
|
return 0;
|
|
}
|
|
Py_ssize_t len = strlen(str);
|
|
if (col_offset > len + 1) {
|
|
col_offset = len + 1;
|
|
}
|
|
assert(col_offset >= 0);
|
|
PyObject *text = PyUnicode_DecodeUTF8(str, col_offset, "replace");
|
|
if (!text) {
|
|
return 0;
|
|
}
|
|
Py_ssize_t size = PyUnicode_GET_LENGTH(text);
|
|
Py_DECREF(text);
|
|
return size;
|
|
}
|
|
|
|
const char *
|
|
_PyPegen_get_expr_name(expr_ty e)
|
|
{
|
|
assert(e != NULL);
|
|
switch (e->kind) {
|
|
case Attribute_kind:
|
|
return "attribute";
|
|
case Subscript_kind:
|
|
return "subscript";
|
|
case Starred_kind:
|
|
return "starred";
|
|
case Name_kind:
|
|
return "name";
|
|
case List_kind:
|
|
return "list";
|
|
case Tuple_kind:
|
|
return "tuple";
|
|
case Lambda_kind:
|
|
return "lambda";
|
|
case Call_kind:
|
|
return "function call";
|
|
case BoolOp_kind:
|
|
case BinOp_kind:
|
|
case UnaryOp_kind:
|
|
return "expression";
|
|
case GeneratorExp_kind:
|
|
return "generator expression";
|
|
case Yield_kind:
|
|
case YieldFrom_kind:
|
|
return "yield expression";
|
|
case Await_kind:
|
|
return "await expression";
|
|
case ListComp_kind:
|
|
return "list comprehension";
|
|
case SetComp_kind:
|
|
return "set comprehension";
|
|
case DictComp_kind:
|
|
return "dict comprehension";
|
|
case Dict_kind:
|
|
return "dict literal";
|
|
case Set_kind:
|
|
return "set display";
|
|
case JoinedStr_kind:
|
|
case FormattedValue_kind:
|
|
return "f-string expression";
|
|
case Constant_kind: {
|
|
PyObject *value = e->v.Constant.value;
|
|
if (value == Py_None) {
|
|
return "None";
|
|
}
|
|
if (value == Py_False) {
|
|
return "False";
|
|
}
|
|
if (value == Py_True) {
|
|
return "True";
|
|
}
|
|
if (value == Py_Ellipsis) {
|
|
return "Ellipsis";
|
|
}
|
|
return "literal";
|
|
}
|
|
case Compare_kind:
|
|
return "comparison";
|
|
case IfExp_kind:
|
|
return "conditional expression";
|
|
case NamedExpr_kind:
|
|
return "named expression";
|
|
default:
|
|
PyErr_Format(PyExc_SystemError,
|
|
"unexpected expression in assignment %d (line %d)",
|
|
e->kind, e->lineno);
|
|
return NULL;
|
|
}
|
|
}
|
|
|
|
static int
|
|
raise_decode_error(Parser *p)
|
|
{
|
|
assert(PyErr_Occurred());
|
|
const char *errtype = NULL;
|
|
if (PyErr_ExceptionMatches(PyExc_UnicodeError)) {
|
|
errtype = "unicode error";
|
|
}
|
|
else if (PyErr_ExceptionMatches(PyExc_ValueError)) {
|
|
errtype = "value error";
|
|
}
|
|
if (errtype) {
|
|
PyObject *type;
|
|
PyObject *value;
|
|
PyObject *tback;
|
|
PyObject *errstr;
|
|
PyErr_Fetch(&type, &value, &tback);
|
|
errstr = PyObject_Str(value);
|
|
if (errstr) {
|
|
RAISE_SYNTAX_ERROR("(%s) %U", errtype, errstr);
|
|
Py_DECREF(errstr);
|
|
}
|
|
else {
|
|
PyErr_Clear();
|
|
RAISE_SYNTAX_ERROR("(%s) unknown error", errtype);
|
|
}
|
|
Py_XDECREF(type);
|
|
Py_XDECREF(value);
|
|
Py_XDECREF(tback);
|
|
}
|
|
|
|
return -1;
|
|
}
|
|
|
|
static inline void
|
|
raise_unclosed_parentheses_error(Parser *p) {
|
|
int error_lineno = p->tok->parenlinenostack[p->tok->level-1];
|
|
int error_col = p->tok->parencolstack[p->tok->level-1];
|
|
RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError,
|
|
error_lineno, error_col, error_lineno, -1,
|
|
"'%c' was never closed",
|
|
p->tok->parenstack[p->tok->level-1]);
|
|
}
|
|
|
|
static void
|
|
raise_tokenizer_init_error(PyObject *filename)
|
|
{
|
|
if (!(PyErr_ExceptionMatches(PyExc_LookupError)
|
|
|| PyErr_ExceptionMatches(PyExc_ValueError)
|
|
|| PyErr_ExceptionMatches(PyExc_UnicodeDecodeError))) {
|
|
return;
|
|
}
|
|
PyObject *errstr = NULL;
|
|
PyObject *tuple = NULL;
|
|
PyObject *type;
|
|
PyObject *value;
|
|
PyObject *tback;
|
|
PyErr_Fetch(&type, &value, &tback);
|
|
errstr = PyObject_Str(value);
|
|
if (!errstr) {
|
|
goto error;
|
|
}
|
|
|
|
PyObject *tmp = Py_BuildValue("(OiiO)", filename, 0, -1, Py_None);
|
|
if (!tmp) {
|
|
goto error;
|
|
}
|
|
|
|
tuple = PyTuple_Pack(2, errstr, tmp);
|
|
Py_DECREF(tmp);
|
|
if (!value) {
|
|
goto error;
|
|
}
|
|
PyErr_SetObject(PyExc_SyntaxError, tuple);
|
|
|
|
error:
|
|
Py_XDECREF(type);
|
|
Py_XDECREF(value);
|
|
Py_XDECREF(tback);
|
|
Py_XDECREF(errstr);
|
|
Py_XDECREF(tuple);
|
|
}
|
|
|
|
static int
|
|
tokenizer_error(Parser *p)
|
|
{
|
|
if (PyErr_Occurred()) {
|
|
return -1;
|
|
}
|
|
|
|
const char *msg = NULL;
|
|
PyObject* errtype = PyExc_SyntaxError;
|
|
Py_ssize_t col_offset = -1;
|
|
switch (p->tok->done) {
|
|
case E_TOKEN:
|
|
msg = "invalid token";
|
|
break;
|
|
case E_EOF:
|
|
if (p->tok->level) {
|
|
raise_unclosed_parentheses_error(p);
|
|
} else {
|
|
RAISE_SYNTAX_ERROR("unexpected EOF while parsing");
|
|
}
|
|
return -1;
|
|
case E_DEDENT:
|
|
RAISE_INDENTATION_ERROR("unindent does not match any outer indentation level");
|
|
return -1;
|
|
case E_INTR:
|
|
if (!PyErr_Occurred()) {
|
|
PyErr_SetNone(PyExc_KeyboardInterrupt);
|
|
}
|
|
return -1;
|
|
case E_NOMEM:
|
|
PyErr_NoMemory();
|
|
return -1;
|
|
case E_TABSPACE:
|
|
errtype = PyExc_TabError;
|
|
msg = "inconsistent use of tabs and spaces in indentation";
|
|
break;
|
|
case E_TOODEEP:
|
|
errtype = PyExc_IndentationError;
|
|
msg = "too many levels of indentation";
|
|
break;
|
|
case E_LINECONT:
|
|
col_offset = strlen(strtok(p->tok->buf, "\n")) - 1;
|
|
msg = "unexpected character after line continuation character";
|
|
break;
|
|
default:
|
|
msg = "unknown parsing error";
|
|
}
|
|
|
|
RAISE_ERROR_KNOWN_LOCATION(p, errtype, p->tok->lineno, col_offset, p->tok->lineno, -1, msg);
|
|
return -1;
|
|
}
|
|
|
|
void *
|
|
_PyPegen_raise_error(Parser *p, PyObject *errtype, const char *errmsg, ...)
|
|
{
|
|
Token *t = p->known_err_token != NULL ? p->known_err_token : p->tokens[p->fill - 1];
|
|
Py_ssize_t col_offset;
|
|
Py_ssize_t end_col_offset = -1;
|
|
if (t->col_offset == -1) {
|
|
col_offset = Py_SAFE_DOWNCAST(p->tok->cur - p->tok->buf,
|
|
intptr_t, int);
|
|
} else {
|
|
col_offset = t->col_offset + 1;
|
|
}
|
|
|
|
if (t->end_col_offset != -1) {
|
|
end_col_offset = t->end_col_offset + 1;
|
|
}
|
|
|
|
va_list va;
|
|
va_start(va, errmsg);
|
|
_PyPegen_raise_error_known_location(p, errtype, t->lineno, col_offset, t->end_lineno, end_col_offset, errmsg, va);
|
|
va_end(va);
|
|
|
|
return NULL;
|
|
}
|
|
|
|
static PyObject *
|
|
get_error_line(Parser *p, Py_ssize_t lineno)
|
|
{
|
|
/* If the file descriptor is interactive, the source lines of the current
|
|
* (multi-line) statement are stored in p->tok->interactive_src_start.
|
|
* If not, we're parsing from a string, which means that the whole source
|
|
* is stored in p->tok->str. */
|
|
assert(p->tok->fp == NULL || p->tok->fp == stdin);
|
|
|
|
char *cur_line = p->tok->fp_interactive ? p->tok->interactive_src_start : p->tok->str;
|
|
|
|
for (int i = 0; i < lineno - 1; i++) {
|
|
cur_line = strchr(cur_line, '\n') + 1;
|
|
}
|
|
|
|
char *next_newline;
|
|
if ((next_newline = strchr(cur_line, '\n')) == NULL) { // This is the last line
|
|
next_newline = cur_line + strlen(cur_line);
|
|
}
|
|
return PyUnicode_DecodeUTF8(cur_line, next_newline - cur_line, "replace");
|
|
}
|
|
|
|
void *
|
|
_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
|
|
Py_ssize_t lineno, Py_ssize_t col_offset,
|
|
Py_ssize_t end_lineno, Py_ssize_t end_col_offset,
|
|
const char *errmsg, va_list va)
|
|
{
|
|
PyObject *value = NULL;
|
|
PyObject *errstr = NULL;
|
|
PyObject *error_line = NULL;
|
|
PyObject *tmp = NULL;
|
|
p->error_indicator = 1;
|
|
|
|
if (end_lineno == CURRENT_POS) {
|
|
end_lineno = p->tok->lineno;
|
|
}
|
|
if (end_col_offset == CURRENT_POS) {
|
|
end_col_offset = p->tok->cur - p->tok->line_start;
|
|
}
|
|
|
|
if (p->start_rule == Py_fstring_input) {
|
|
const char *fstring_msg = "f-string: ";
|
|
Py_ssize_t len = strlen(fstring_msg) + strlen(errmsg);
|
|
|
|
char *new_errmsg = PyMem_Malloc(len + 1); // Lengths of both strings plus NULL character
|
|
if (!new_errmsg) {
|
|
return (void *) PyErr_NoMemory();
|
|
}
|
|
|
|
// Copy both strings into new buffer
|
|
memcpy(new_errmsg, fstring_msg, strlen(fstring_msg));
|
|
memcpy(new_errmsg + strlen(fstring_msg), errmsg, strlen(errmsg));
|
|
new_errmsg[len] = 0;
|
|
errmsg = new_errmsg;
|
|
}
|
|
errstr = PyUnicode_FromFormatV(errmsg, va);
|
|
if (!errstr) {
|
|
goto error;
|
|
}
|
|
|
|
if (p->tok->fp_interactive) {
|
|
error_line = get_error_line(p, lineno);
|
|
}
|
|
else if (p->start_rule == Py_file_input) {
|
|
error_line = PyErr_ProgramTextObject(p->tok->filename, (int) lineno);
|
|
}
|
|
|
|
if (!error_line) {
|
|
/* PyErr_ProgramTextObject was not called or returned NULL. If it was not called,
|
|
then we need to find the error line from some other source, because
|
|
p->start_rule != Py_file_input. If it returned NULL, then it either unexpectedly
|
|
failed or we're parsing from a string or the REPL. There's a third edge case where
|
|
we're actually parsing from a file, which has an E_EOF SyntaxError and in that case
|
|
`PyErr_ProgramTextObject` fails because lineno points to last_file_line + 1, which
|
|
does not physically exist */
|
|
assert(p->tok->fp == NULL || p->tok->fp == stdin || p->tok->done == E_EOF);
|
|
|
|
if (p->tok->lineno <= lineno) {
|
|
Py_ssize_t size = p->tok->inp - p->tok->buf;
|
|
error_line = PyUnicode_DecodeUTF8(p->tok->buf, size, "replace");
|
|
}
|
|
else {
|
|
error_line = get_error_line(p, lineno);
|
|
}
|
|
if (!error_line) {
|
|
goto error;
|
|
}
|
|
}
|
|
|
|
if (p->start_rule == Py_fstring_input) {
|
|
col_offset -= p->starting_col_offset;
|
|
end_col_offset -= p->starting_col_offset;
|
|
}
|
|
|
|
Py_ssize_t col_number = col_offset;
|
|
Py_ssize_t end_col_number = end_col_offset;
|
|
|
|
if (p->tok->encoding != NULL) {
|
|
col_number = byte_offset_to_character_offset(error_line, col_offset);
|
|
end_col_number = end_col_number > 0 ?
|
|
byte_offset_to_character_offset(error_line, end_col_offset) :
|
|
end_col_number;
|
|
}
|
|
tmp = Py_BuildValue("(OiiNii)", p->tok->filename, lineno, col_number, error_line, end_lineno, end_col_number);
|
|
if (!tmp) {
|
|
goto error;
|
|
}
|
|
value = PyTuple_Pack(2, errstr, tmp);
|
|
Py_DECREF(tmp);
|
|
if (!value) {
|
|
goto error;
|
|
}
|
|
PyErr_SetObject(errtype, value);
|
|
|
|
Py_DECREF(errstr);
|
|
Py_DECREF(value);
|
|
if (p->start_rule == Py_fstring_input) {
|
|
PyMem_Free((void *)errmsg);
|
|
}
|
|
return NULL;
|
|
|
|
error:
|
|
Py_XDECREF(errstr);
|
|
Py_XDECREF(error_line);
|
|
if (p->start_rule == Py_fstring_input) {
|
|
PyMem_Free((void *)errmsg);
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
#if 0
|
|
static const char *
|
|
token_name(int type)
|
|
{
|
|
if (0 <= type && type <= N_TOKENS) {
|
|
return _PyParser_TokenNames[type];
|
|
}
|
|
return "<Huh?>";
|
|
}
|
|
#endif
|
|
|
|
// Here, mark is the start of the node, while p->mark is the end.
|
|
// If node==NULL, they should be the same.
|
|
int
|
|
_PyPegen_insert_memo(Parser *p, int mark, int type, void *node)
|
|
{
|
|
// Insert in front
|
|
Memo *m = _PyArena_Malloc(p->arena, sizeof(Memo));
|
|
if (m == NULL) {
|
|
return -1;
|
|
}
|
|
m->type = type;
|
|
m->node = node;
|
|
m->mark = p->mark;
|
|
m->next = p->tokens[mark]->memo;
|
|
p->tokens[mark]->memo = m;
|
|
return 0;
|
|
}
|
|
|
|
// Like _PyPegen_insert_memo(), but updates an existing node if found.
|
|
int
|
|
_PyPegen_update_memo(Parser *p, int mark, int type, void *node)
|
|
{
|
|
for (Memo *m = p->tokens[mark]->memo; m != NULL; m = m->next) {
|
|
if (m->type == type) {
|
|
// Update existing node.
|
|
m->node = node;
|
|
m->mark = p->mark;
|
|
return 0;
|
|
}
|
|
}
|
|
// Insert new node.
|
|
return _PyPegen_insert_memo(p, mark, type, node);
|
|
}
|
|
|
|
// Return dummy NAME.
|
|
void *
|
|
_PyPegen_dummy_name(Parser *p, ...)
|
|
{
|
|
static void *cache = NULL;
|
|
|
|
if (cache != NULL) {
|
|
return cache;
|
|
}
|
|
|
|
PyObject *id = _create_dummy_identifier(p);
|
|
if (!id) {
|
|
return NULL;
|
|
}
|
|
cache = _PyAST_Name(id, Load, 1, 0, 1, 0, p->arena);
|
|
return cache;
|
|
}
|
|
|
|
static int
|
|
_get_keyword_or_name_type(Parser *p, const char *name, int name_len)
|
|
{
|
|
assert(name_len > 0);
|
|
if (name_len >= p->n_keyword_lists ||
|
|
p->keywords[name_len] == NULL ||
|
|
p->keywords[name_len]->type == -1) {
|
|
return NAME;
|
|
}
|
|
for (KeywordToken *k = p->keywords[name_len]; k != NULL && k->type != -1; k++) {
|
|
if (strncmp(k->str, name, name_len) == 0) {
|
|
return k->type;
|
|
}
|
|
}
|
|
return NAME;
|
|
}
|
|
|
|
static int
|
|
growable_comment_array_init(growable_comment_array *arr, size_t initial_size) {
|
|
assert(initial_size > 0);
|
|
arr->items = PyMem_Malloc(initial_size * sizeof(*arr->items));
|
|
arr->size = initial_size;
|
|
arr->num_items = 0;
|
|
|
|
return arr->items != NULL;
|
|
}
|
|
|
|
static int
|
|
growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) {
|
|
if (arr->num_items >= arr->size) {
|
|
size_t new_size = arr->size * 2;
|
|
void *new_items_array = PyMem_Realloc(arr->items, new_size * sizeof(*arr->items));
|
|
if (!new_items_array) {
|
|
return 0;
|
|
}
|
|
arr->items = new_items_array;
|
|
arr->size = new_size;
|
|
}
|
|
|
|
arr->items[arr->num_items].lineno = lineno;
|
|
arr->items[arr->num_items].comment = comment; // Take ownership
|
|
arr->num_items++;
|
|
return 1;
|
|
}
|
|
|
|
static void
|
|
growable_comment_array_deallocate(growable_comment_array *arr) {
|
|
for (unsigned i = 0; i < arr->num_items; i++) {
|
|
PyMem_Free(arr->items[i].comment);
|
|
}
|
|
PyMem_Free(arr->items);
|
|
}
|
|
|
|
static int
|
|
initialize_token(Parser *p, Token *token, const char *start, const char *end, int token_type) {
|
|
assert(token != NULL);
|
|
|
|
token->type = (token_type == NAME) ? _get_keyword_or_name_type(p, start, (int)(end - start)) : token_type;
|
|
token->bytes = PyBytes_FromStringAndSize(start, end - start);
|
|
if (token->bytes == NULL) {
|
|
return -1;
|
|
}
|
|
|
|
if (_PyArena_AddPyObject(p->arena, token->bytes) < 0) {
|
|
Py_DECREF(token->bytes);
|
|
return -1;
|
|
}
|
|
|
|
const char *line_start = token_type == STRING ? p->tok->multi_line_start : p->tok->line_start;
|
|
int lineno = token_type == STRING ? p->tok->first_lineno : p->tok->lineno;
|
|
int end_lineno = p->tok->lineno;
|
|
|
|
int col_offset = (start != NULL && start >= line_start) ? (int)(start - line_start) : -1;
|
|
int end_col_offset = (end != NULL && end >= p->tok->line_start) ? (int)(end - p->tok->line_start) : -1;
|
|
|
|
token->lineno = p->starting_lineno + lineno;
|
|
token->col_offset = p->tok->lineno == 1 ? p->starting_col_offset + col_offset : col_offset;
|
|
token->end_lineno = p->starting_lineno + end_lineno;
|
|
token->end_col_offset = p->tok->lineno == 1 ? p->starting_col_offset + end_col_offset : end_col_offset;
|
|
|
|
p->fill += 1;
|
|
|
|
if (token_type == ERRORTOKEN && p->tok->done == E_DECODE) {
|
|
return raise_decode_error(p);
|
|
}
|
|
|
|
return (token_type == ERRORTOKEN ? tokenizer_error(p) : 0);
|
|
}
|
|
|
|
static int
|
|
_resize_tokens_array(Parser *p) {
|
|
int newsize = p->size * 2;
|
|
Token **new_tokens = PyMem_Realloc(p->tokens, newsize * sizeof(Token *));
|
|
if (new_tokens == NULL) {
|
|
PyErr_NoMemory();
|
|
return -1;
|
|
}
|
|
p->tokens = new_tokens;
|
|
|
|
for (int i = p->size; i < newsize; i++) {
|
|
p->tokens[i] = PyMem_Calloc(1, sizeof(Token));
|
|
if (p->tokens[i] == NULL) {
|
|
p->size = i; // Needed, in order to cleanup correctly after parser fails
|
|
PyErr_NoMemory();
|
|
return -1;
|
|
}
|
|
}
|
|
p->size = newsize;
|
|
return 0;
|
|
}
|
|
|
|
int
|
|
_PyPegen_fill_token(Parser *p)
|
|
{
|
|
const char *start;
|
|
const char *end;
|
|
int type = PyTokenizer_Get(p->tok, &start, &end);
|
|
|
|
// Record and skip '# type: ignore' comments
|
|
while (type == TYPE_IGNORE) {
|
|
Py_ssize_t len = end - start;
|
|
char *tag = PyMem_Malloc(len + 1);
|
|
if (tag == NULL) {
|
|
PyErr_NoMemory();
|
|
return -1;
|
|
}
|
|
strncpy(tag, start, len);
|
|
tag[len] = '\0';
|
|
// Ownership of tag passes to the growable array
|
|
if (!growable_comment_array_add(&p->type_ignore_comments, p->tok->lineno, tag)) {
|
|
PyErr_NoMemory();
|
|
return -1;
|
|
}
|
|
type = PyTokenizer_Get(p->tok, &start, &end);
|
|
}
|
|
|
|
// If we have reached the end and we are in single input mode we need to insert a newline and reset the parsing
|
|
if (p->start_rule == Py_single_input && type == ENDMARKER && p->parsing_started) {
|
|
type = NEWLINE; /* Add an extra newline */
|
|
p->parsing_started = 0;
|
|
|
|
if (p->tok->indent && !(p->flags & PyPARSE_DONT_IMPLY_DEDENT)) {
|
|
p->tok->pendin = -p->tok->indent;
|
|
p->tok->indent = 0;
|
|
}
|
|
}
|
|
else {
|
|
p->parsing_started = 1;
|
|
}
|
|
|
|
// Check if we are at the limit of the token array capacity and resize if needed
|
|
if ((p->fill == p->size) && (_resize_tokens_array(p) != 0)) {
|
|
return -1;
|
|
}
|
|
|
|
Token *t = p->tokens[p->fill];
|
|
return initialize_token(p, t, start, end, type);
|
|
}
|
|
|
|
|
|
#if defined(Py_DEBUG)
|
|
// Instrumentation to count the effectiveness of memoization.
|
|
// The array counts the number of tokens skipped by memoization,
|
|
// indexed by type.
|
|
|
|
#define NSTATISTICS 2000
|
|
static long memo_statistics[NSTATISTICS];
|
|
|
|
void
|
|
_PyPegen_clear_memo_statistics()
|
|
{
|
|
for (int i = 0; i < NSTATISTICS; i++) {
|
|
memo_statistics[i] = 0;
|
|
}
|
|
}
|
|
|
|
PyObject *
|
|
_PyPegen_get_memo_statistics()
|
|
{
|
|
PyObject *ret = PyList_New(NSTATISTICS);
|
|
if (ret == NULL) {
|
|
return NULL;
|
|
}
|
|
for (int i = 0; i < NSTATISTICS; i++) {
|
|
PyObject *value = PyLong_FromLong(memo_statistics[i]);
|
|
if (value == NULL) {
|
|
Py_DECREF(ret);
|
|
return NULL;
|
|
}
|
|
// PyList_SetItem borrows a reference to value.
|
|
if (PyList_SetItem(ret, i, value) < 0) {
|
|
Py_DECREF(ret);
|
|
return NULL;
|
|
}
|
|
}
|
|
return ret;
|
|
}
|
|
#endif
|
|
|
|
int // bool
|
|
_PyPegen_is_memoized(Parser *p, int type, void *pres)
|
|
{
|
|
if (p->mark == p->fill) {
|
|
if (_PyPegen_fill_token(p) < 0) {
|
|
p->error_indicator = 1;
|
|
return -1;
|
|
}
|
|
}
|
|
|
|
Token *t = p->tokens[p->mark];
|
|
|
|
for (Memo *m = t->memo; m != NULL; m = m->next) {
|
|
if (m->type == type) {
|
|
#if defined(PY_DEBUG)
|
|
if (0 <= type && type < NSTATISTICS) {
|
|
long count = m->mark - p->mark;
|
|
// A memoized negative result counts for one.
|
|
if (count <= 0) {
|
|
count = 1;
|
|
}
|
|
memo_statistics[type] += count;
|
|
}
|
|
#endif
|
|
p->mark = m->mark;
|
|
*(void **)(pres) = m->node;
|
|
return 1;
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
int
|
|
_PyPegen_lookahead_with_name(int positive, expr_ty (func)(Parser *), Parser *p)
|
|
{
|
|
int mark = p->mark;
|
|
void *res = func(p);
|
|
p->mark = mark;
|
|
return (res != NULL) == positive;
|
|
}
|
|
|
|
int
|
|
_PyPegen_lookahead_with_string(int positive, expr_ty (func)(Parser *, const char*), Parser *p, const char* arg)
|
|
{
|
|
int mark = p->mark;
|
|
void *res = func(p, arg);
|
|
p->mark = mark;
|
|
return (res != NULL) == positive;
|
|
}
|
|
|
|
int
|
|
_PyPegen_lookahead_with_int(int positive, Token *(func)(Parser *, int), Parser *p, int arg)
|
|
{
|
|
int mark = p->mark;
|
|
void *res = func(p, arg);
|
|
p->mark = mark;
|
|
return (res != NULL) == positive;
|
|
}
|
|
|
|
int
|
|
_PyPegen_lookahead(int positive, void *(func)(Parser *), Parser *p)
|
|
{
|
|
int mark = p->mark;
|
|
void *res = (void*)func(p);
|
|
p->mark = mark;
|
|
return (res != NULL) == positive;
|
|
}
|
|
|
|
Token *
|
|
_PyPegen_expect_token(Parser *p, int type)
|
|
{
|
|
if (p->mark == p->fill) {
|
|
if (_PyPegen_fill_token(p) < 0) {
|
|
p->error_indicator = 1;
|
|
return NULL;
|
|
}
|
|
}
|
|
Token *t = p->tokens[p->mark];
|
|
if (t->type != type) {
|
|
return NULL;
|
|
}
|
|
p->mark += 1;
|
|
return t;
|
|
}
|
|
|
|
Token *
|
|
_PyPegen_expect_forced_token(Parser *p, int type, const char* expected) {
|
|
|
|
if (p->error_indicator == 1) {
|
|
return NULL;
|
|
}
|
|
|
|
if (p->mark == p->fill) {
|
|
if (_PyPegen_fill_token(p) < 0) {
|
|
p->error_indicator = 1;
|
|
return NULL;
|
|
}
|
|
}
|
|
Token *t = p->tokens[p->mark];
|
|
if (t->type != type) {
|
|
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(t, "expected '%s'", expected);
|
|
return NULL;
|
|
}
|
|
p->mark += 1;
|
|
return t;
|
|
}
|
|
|
|
expr_ty
|
|
_PyPegen_expect_soft_keyword(Parser *p, const char *keyword)
|
|
{
|
|
if (p->mark == p->fill) {
|
|
if (_PyPegen_fill_token(p) < 0) {
|
|
p->error_indicator = 1;
|
|
return NULL;
|
|
}
|
|
}
|
|
Token *t = p->tokens[p->mark];
|
|
if (t->type != NAME) {
|
|
return NULL;
|
|
}
|
|
char *s = PyBytes_AsString(t->bytes);
|
|
if (!s) {
|
|
p->error_indicator = 1;
|
|
return NULL;
|
|
}
|
|
if (strcmp(s, keyword) != 0) {
|
|
return NULL;
|
|
}
|
|
return _PyPegen_name_token(p);
|
|
}
|
|
|
|
Token *
|
|
_PyPegen_get_last_nonnwhitespace_token(Parser *p)
|
|
{
|
|
assert(p->mark >= 0);
|
|
Token *token = NULL;
|
|
for (int m = p->mark - 1; m >= 0; m--) {
|
|
token = p->tokens[m];
|
|
if (token->type != ENDMARKER && (token->type < NEWLINE || token->type > DEDENT)) {
|
|
break;
|
|
}
|
|
}
|
|
return token;
|
|
}
|
|
|
|
expr_ty
|
|
_PyPegen_name_token(Parser *p)
|
|
{
|
|
Token *t = _PyPegen_expect_token(p, NAME);
|
|
if (t == NULL) {
|
|
return NULL;
|
|
}
|
|
char* s = PyBytes_AsString(t->bytes);
|
|
if (!s) {
|
|
p->error_indicator = 1;
|
|
return NULL;
|
|
}
|
|
PyObject *id = _PyPegen_new_identifier(p, s);
|
|
if (id == NULL) {
|
|
p->error_indicator = 1;
|
|
return NULL;
|
|
}
|
|
return _PyAST_Name(id, Load, t->lineno, t->col_offset, t->end_lineno,
|
|
t->end_col_offset, p->arena);
|
|
}
|
|
|
|
void *
|
|
_PyPegen_string_token(Parser *p)
|
|
{
|
|
return _PyPegen_expect_token(p, STRING);
|
|
}
|
|
|
|
|
|
expr_ty _PyPegen_soft_keyword_token(Parser *p) {
|
|
Token *t = _PyPegen_expect_token(p, NAME);
|
|
if (t == NULL) {
|
|
return NULL;
|
|
}
|
|
char *the_token;
|
|
Py_ssize_t size;
|
|
PyBytes_AsStringAndSize(t->bytes, &the_token, &size);
|
|
for (char **keyword = p->soft_keywords; *keyword != NULL; keyword++) {
|
|
if (strncmp(*keyword, the_token, size) == 0) {
|
|
return _PyPegen_name_token(p);
|
|
}
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
static PyObject *
|
|
parsenumber_raw(const char *s)
|
|
{
|
|
const char *end;
|
|
long x;
|
|
double dx;
|
|
Py_complex compl;
|
|
int imflag;
|
|
|
|
assert(s != NULL);
|
|
errno = 0;
|
|
end = s + strlen(s) - 1;
|
|
imflag = *end == 'j' || *end == 'J';
|
|
if (s[0] == '0') {
|
|
x = (long)PyOS_strtoul(s, (char **)&end, 0);
|
|
if (x < 0 && errno == 0) {
|
|
return PyLong_FromString(s, (char **)0, 0);
|
|
}
|
|
}
|
|
else {
|
|
x = PyOS_strtol(s, (char **)&end, 0);
|
|
}
|
|
if (*end == '\0') {
|
|
if (errno != 0) {
|
|
return PyLong_FromString(s, (char **)0, 0);
|
|
}
|
|
return PyLong_FromLong(x);
|
|
}
|
|
/* XXX Huge floats may silently fail */
|
|
if (imflag) {
|
|
compl.real = 0.;
|
|
compl.imag = PyOS_string_to_double(s, (char **)&end, NULL);
|
|
if (compl.imag == -1.0 && PyErr_Occurred()) {
|
|
return NULL;
|
|
}
|
|
return PyComplex_FromCComplex(compl);
|
|
}
|
|
dx = PyOS_string_to_double(s, NULL, NULL);
|
|
if (dx == -1.0 && PyErr_Occurred()) {
|
|
return NULL;
|
|
}
|
|
return PyFloat_FromDouble(dx);
|
|
}
|
|
|
|
static PyObject *
|
|
parsenumber(const char *s)
|
|
{
|
|
char *dup;
|
|
char *end;
|
|
PyObject *res = NULL;
|
|
|
|
assert(s != NULL);
|
|
|
|
if (strchr(s, '_') == NULL) {
|
|
return parsenumber_raw(s);
|
|
}
|
|
/* Create a duplicate without underscores. */
|
|
dup = PyMem_Malloc(strlen(s) + 1);
|
|
if (dup == NULL) {
|
|
return PyErr_NoMemory();
|
|
}
|
|
end = dup;
|
|
for (; *s; s++) {
|
|
if (*s != '_') {
|
|
*end++ = *s;
|
|
}
|
|
}
|
|
*end = '\0';
|
|
res = parsenumber_raw(dup);
|
|
PyMem_Free(dup);
|
|
return res;
|
|
}
|
|
|
|
expr_ty
|
|
_PyPegen_number_token(Parser *p)
|
|
{
|
|
Token *t = _PyPegen_expect_token(p, NUMBER);
|
|
if (t == NULL) {
|
|
return NULL;
|
|
}
|
|
|
|
char *num_raw = PyBytes_AsString(t->bytes);
|
|
if (num_raw == NULL) {
|
|
p->error_indicator = 1;
|
|
return NULL;
|
|
}
|
|
|
|
if (p->feature_version < 6 && strchr(num_raw, '_') != NULL) {
|
|
p->error_indicator = 1;
|
|
return RAISE_SYNTAX_ERROR("Underscores in numeric literals are only supported "
|
|
"in Python 3.6 and greater");
|
|
}
|
|
|
|
PyObject *c = parsenumber(num_raw);
|
|
|
|
if (c == NULL) {
|
|
p->error_indicator = 1;
|
|
return NULL;
|
|
}
|
|
|
|
if (_PyArena_AddPyObject(p->arena, c) < 0) {
|
|
Py_DECREF(c);
|
|
p->error_indicator = 1;
|
|
return NULL;
|
|
}
|
|
|
|
return _PyAST_Constant(c, NULL, t->lineno, t->col_offset, t->end_lineno,
|
|
t->end_col_offset, p->arena);
|
|
}
|
|
|
|
static int // bool
|
|
newline_in_string(Parser *p, const char *cur)
|
|
{
|
|
for (const char *c = cur; c >= p->tok->buf; c--) {
|
|
if (*c == '\'' || *c == '"') {
|
|
return 1;
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
/* Check that the source for a single input statement really is a single
|
|
statement by looking at what is left in the buffer after parsing.
|
|
Trailing whitespace and comments are OK. */
|
|
static int // bool
|
|
bad_single_statement(Parser *p)
|
|
{
|
|
const char *cur = strchr(p->tok->buf, '\n');
|
|
|
|
/* Newlines are allowed if preceded by a line continuation character
|
|
or if they appear inside a string. */
|
|
if (!cur || (cur != p->tok->buf && *(cur - 1) == '\\')
|
|
|| newline_in_string(p, cur)) {
|
|
return 0;
|
|
}
|
|
char c = *cur;
|
|
|
|
for (;;) {
|
|
while (c == ' ' || c == '\t' || c == '\n' || c == '\014') {
|
|
c = *++cur;
|
|
}
|
|
|
|
if (!c) {
|
|
return 0;
|
|
}
|
|
|
|
if (c != '#') {
|
|
return 1;
|
|
}
|
|
|
|
/* Suck up comment. */
|
|
while (c && c != '\n') {
|
|
c = *++cur;
|
|
}
|
|
}
|
|
}
|
|
|
|
void
|
|
_PyPegen_Parser_Free(Parser *p)
|
|
{
|
|
Py_XDECREF(p->normalize);
|
|
for (int i = 0; i < p->size; i++) {
|
|
PyMem_Free(p->tokens[i]);
|
|
}
|
|
PyMem_Free(p->tokens);
|
|
growable_comment_array_deallocate(&p->type_ignore_comments);
|
|
PyMem_Free(p);
|
|
}
|
|
|
|
static int
|
|
compute_parser_flags(PyCompilerFlags *flags)
|
|
{
|
|
int parser_flags = 0;
|
|
if (!flags) {
|
|
return 0;
|
|
}
|
|
if (flags->cf_flags & PyCF_DONT_IMPLY_DEDENT) {
|
|
parser_flags |= PyPARSE_DONT_IMPLY_DEDENT;
|
|
}
|
|
if (flags->cf_flags & PyCF_IGNORE_COOKIE) {
|
|
parser_flags |= PyPARSE_IGNORE_COOKIE;
|
|
}
|
|
if (flags->cf_flags & CO_FUTURE_BARRY_AS_BDFL) {
|
|
parser_flags |= PyPARSE_BARRY_AS_BDFL;
|
|
}
|
|
if (flags->cf_flags & PyCF_TYPE_COMMENTS) {
|
|
parser_flags |= PyPARSE_TYPE_COMMENTS;
|
|
}
|
|
if ((flags->cf_flags & PyCF_ONLY_AST) && flags->cf_feature_version < 7) {
|
|
parser_flags |= PyPARSE_ASYNC_HACKS;
|
|
}
|
|
return parser_flags;
|
|
}
|
|
|
|
Parser *
|
|
_PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
|
|
int feature_version, int *errcode, PyArena *arena)
|
|
{
|
|
Parser *p = PyMem_Malloc(sizeof(Parser));
|
|
if (p == NULL) {
|
|
return (Parser *) PyErr_NoMemory();
|
|
}
|
|
assert(tok != NULL);
|
|
tok->type_comments = (flags & PyPARSE_TYPE_COMMENTS) > 0;
|
|
tok->async_hacks = (flags & PyPARSE_ASYNC_HACKS) > 0;
|
|
p->tok = tok;
|
|
p->keywords = NULL;
|
|
p->n_keyword_lists = -1;
|
|
p->soft_keywords = NULL;
|
|
p->tokens = PyMem_Malloc(sizeof(Token *));
|
|
if (!p->tokens) {
|
|
PyMem_Free(p);
|
|
return (Parser *) PyErr_NoMemory();
|
|
}
|
|
p->tokens[0] = PyMem_Calloc(1, sizeof(Token));
|
|
if (!p->tokens) {
|
|
PyMem_Free(p->tokens);
|
|
PyMem_Free(p);
|
|
return (Parser *) PyErr_NoMemory();
|
|
}
|
|
if (!growable_comment_array_init(&p->type_ignore_comments, 10)) {
|
|
PyMem_Free(p->tokens[0]);
|
|
PyMem_Free(p->tokens);
|
|
PyMem_Free(p);
|
|
return (Parser *) PyErr_NoMemory();
|
|
}
|
|
|
|
p->mark = 0;
|
|
p->fill = 0;
|
|
p->size = 1;
|
|
|
|
p->errcode = errcode;
|
|
p->arena = arena;
|
|
p->start_rule = start_rule;
|
|
p->parsing_started = 0;
|
|
p->normalize = NULL;
|
|
p->error_indicator = 0;
|
|
|
|
p->starting_lineno = 0;
|
|
p->starting_col_offset = 0;
|
|
p->flags = flags;
|
|
p->feature_version = feature_version;
|
|
p->known_err_token = NULL;
|
|
p->level = 0;
|
|
p->call_invalid_rules = 0;
|
|
|
|
return p;
|
|
}
|
|
|
|
static void
|
|
reset_parser_state(Parser *p)
|
|
{
|
|
for (int i = 0; i < p->fill; i++) {
|
|
p->tokens[i]->memo = NULL;
|
|
}
|
|
p->mark = 0;
|
|
p->call_invalid_rules = 1;
|
|
}
|
|
|
|
static int
|
|
_PyPegen_check_tokenizer_errors(Parser *p) {
|
|
// Tokenize the whole input to see if there are any tokenization
|
|
// errors such as mistmatching parentheses. These will get priority
|
|
// over generic syntax errors only if the line number of the error is
|
|
// before the one that we had for the generic error.
|
|
|
|
// We don't want to tokenize to the end for interactive input
|
|
if (p->tok->prompt != NULL) {
|
|
return 0;
|
|
}
|
|
|
|
Token *current_token = p->known_err_token != NULL ? p->known_err_token : p->tokens[p->fill - 1];
|
|
Py_ssize_t current_err_line = current_token->lineno;
|
|
|
|
for (;;) {
|
|
const char *start;
|
|
const char *end;
|
|
switch (PyTokenizer_Get(p->tok, &start, &end)) {
|
|
case ERRORTOKEN:
|
|
if (p->tok->level != 0) {
|
|
int error_lineno = p->tok->parenlinenostack[p->tok->level-1];
|
|
if (current_err_line > error_lineno) {
|
|
raise_unclosed_parentheses_error(p);
|
|
return -1;
|
|
}
|
|
}
|
|
break;
|
|
case ENDMARKER:
|
|
break;
|
|
default:
|
|
continue;
|
|
}
|
|
break;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
void *
|
|
_PyPegen_run_parser(Parser *p)
|
|
{
|
|
void *res = _PyPegen_parse(p);
|
|
if (res == NULL) {
|
|
reset_parser_state(p);
|
|
_PyPegen_parse(p);
|
|
if (PyErr_Occurred()) {
|
|
return NULL;
|
|
}
|
|
if (p->fill == 0) {
|
|
RAISE_SYNTAX_ERROR("error at start before reading any input");
|
|
}
|
|
else if (p->tok->done == E_EOF) {
|
|
if (p->tok->level) {
|
|
raise_unclosed_parentheses_error(p);
|
|
} else {
|
|
RAISE_SYNTAX_ERROR("unexpected EOF while parsing");
|
|
}
|
|
}
|
|
else {
|
|
if (p->tokens[p->fill-1]->type == INDENT) {
|
|
RAISE_INDENTATION_ERROR("unexpected indent");
|
|
}
|
|
else if (p->tokens[p->fill-1]->type == DEDENT) {
|
|
RAISE_INDENTATION_ERROR("unexpected unindent");
|
|
}
|
|
else {
|
|
RAISE_SYNTAX_ERROR("invalid syntax");
|
|
// _PyPegen_check_tokenizer_errors will override the existing
|
|
// generic SyntaxError we just raised if errors are found.
|
|
_PyPegen_check_tokenizer_errors(p);
|
|
}
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
if (p->start_rule == Py_single_input && bad_single_statement(p)) {
|
|
p->tok->done = E_BADSINGLE; // This is not necessary for now, but might be in the future
|
|
return RAISE_SYNTAX_ERROR("multiple statements found while compiling a single statement");
|
|
}
|
|
|
|
// test_peg_generator defines _Py_TEST_PEGEN to not call PyAST_Validate()
|
|
#if defined(Py_DEBUG) && !defined(_Py_TEST_PEGEN)
|
|
if (p->start_rule == Py_single_input ||
|
|
p->start_rule == Py_file_input ||
|
|
p->start_rule == Py_eval_input)
|
|
{
|
|
if (!_PyAST_Validate(res)) {
|
|
return NULL;
|
|
}
|
|
}
|
|
#endif
|
|
return res;
|
|
}
|
|
|
|
mod_ty
|
|
_PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filename_ob,
|
|
const char *enc, const char *ps1, const char *ps2,
|
|
PyCompilerFlags *flags, int *errcode, PyArena *arena)
|
|
{
|
|
struct tok_state *tok = PyTokenizer_FromFile(fp, enc, ps1, ps2);
|
|
if (tok == NULL) {
|
|
if (PyErr_Occurred()) {
|
|
raise_tokenizer_init_error(filename_ob);
|
|
return NULL;
|
|
}
|
|
return NULL;
|
|
}
|
|
if (!tok->fp || ps1 != NULL || ps2 != NULL ||
|
|
PyUnicode_CompareWithASCIIString(filename_ob, "<stdin>") == 0) {
|
|
tok->fp_interactive = 1;
|
|
}
|
|
// This transfers the ownership to the tokenizer
|
|
tok->filename = filename_ob;
|
|
Py_INCREF(filename_ob);
|
|
|
|
// From here on we need to clean up even if there's an error
|
|
mod_ty result = NULL;
|
|
|
|
int parser_flags = compute_parser_flags(flags);
|
|
Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, PY_MINOR_VERSION,
|
|
errcode, arena);
|
|
if (p == NULL) {
|
|
goto error;
|
|
}
|
|
|
|
result = _PyPegen_run_parser(p);
|
|
_PyPegen_Parser_Free(p);
|
|
|
|
error:
|
|
PyTokenizer_Free(tok);
|
|
return result;
|
|
}
|
|
|
|
mod_ty
|
|
_PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filename_ob,
|
|
PyCompilerFlags *flags, PyArena *arena)
|
|
{
|
|
int exec_input = start_rule == Py_file_input;
|
|
|
|
struct tok_state *tok;
|
|
if (flags == NULL || flags->cf_flags & PyCF_IGNORE_COOKIE) {
|
|
tok = PyTokenizer_FromUTF8(str, exec_input);
|
|
} else {
|
|
tok = PyTokenizer_FromString(str, exec_input);
|
|
}
|
|
if (tok == NULL) {
|
|
if (PyErr_Occurred()) {
|
|
raise_tokenizer_init_error(filename_ob);
|
|
}
|
|
return NULL;
|
|
}
|
|
// This transfers the ownership to the tokenizer
|
|
tok->filename = filename_ob;
|
|
Py_INCREF(filename_ob);
|
|
|
|
// We need to clear up from here on
|
|
mod_ty result = NULL;
|
|
|
|
int parser_flags = compute_parser_flags(flags);
|
|
int feature_version = flags && (flags->cf_flags & PyCF_ONLY_AST) ?
|
|
flags->cf_feature_version : PY_MINOR_VERSION;
|
|
Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, feature_version,
|
|
NULL, arena);
|
|
if (p == NULL) {
|
|
goto error;
|
|
}
|
|
|
|
result = _PyPegen_run_parser(p);
|
|
_PyPegen_Parser_Free(p);
|
|
|
|
error:
|
|
PyTokenizer_Free(tok);
|
|
return result;
|
|
}
|
|
|
|
asdl_stmt_seq*
|
|
_PyPegen_interactive_exit(Parser *p)
|
|
{
|
|
if (p->errcode) {
|
|
*(p->errcode) = E_EOF;
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
/* Creates a single-element asdl_seq* that contains a */
|
|
asdl_seq *
|
|
_PyPegen_singleton_seq(Parser *p, void *a)
|
|
{
|
|
assert(a != NULL);
|
|
asdl_seq *seq = (asdl_seq*)_Py_asdl_generic_seq_new(1, p->arena);
|
|
if (!seq) {
|
|
return NULL;
|
|
}
|
|
asdl_seq_SET_UNTYPED(seq, 0, a);
|
|
return seq;
|
|
}
|
|
|
|
/* Creates a copy of seq and prepends a to it */
|
|
asdl_seq *
|
|
_PyPegen_seq_insert_in_front(Parser *p, void *a, asdl_seq *seq)
|
|
{
|
|
assert(a != NULL);
|
|
if (!seq) {
|
|
return _PyPegen_singleton_seq(p, a);
|
|
}
|
|
|
|
asdl_seq *new_seq = (asdl_seq*)_Py_asdl_generic_seq_new(asdl_seq_LEN(seq) + 1, p->arena);
|
|
if (!new_seq) {
|
|
return NULL;
|
|
}
|
|
|
|
asdl_seq_SET_UNTYPED(new_seq, 0, a);
|
|
for (Py_ssize_t i = 1, l = asdl_seq_LEN(new_seq); i < l; i++) {
|
|
asdl_seq_SET_UNTYPED(new_seq, i, asdl_seq_GET_UNTYPED(seq, i - 1));
|
|
}
|
|
return new_seq;
|
|
}
|
|
|
|
/* Creates a copy of seq and appends a to it */
|
|
asdl_seq *
|
|
_PyPegen_seq_append_to_end(Parser *p, asdl_seq *seq, void *a)
|
|
{
|
|
assert(a != NULL);
|
|
if (!seq) {
|
|
return _PyPegen_singleton_seq(p, a);
|
|
}
|
|
|
|
asdl_seq *new_seq = (asdl_seq*)_Py_asdl_generic_seq_new(asdl_seq_LEN(seq) + 1, p->arena);
|
|
if (!new_seq) {
|
|
return NULL;
|
|
}
|
|
|
|
for (Py_ssize_t i = 0, l = asdl_seq_LEN(new_seq); i + 1 < l; i++) {
|
|
asdl_seq_SET_UNTYPED(new_seq, i, asdl_seq_GET_UNTYPED(seq, i));
|
|
}
|
|
asdl_seq_SET_UNTYPED(new_seq, asdl_seq_LEN(new_seq) - 1, a);
|
|
return new_seq;
|
|
}
|
|
|
|
static Py_ssize_t
|
|
_get_flattened_seq_size(asdl_seq *seqs)
|
|
{
|
|
Py_ssize_t size = 0;
|
|
for (Py_ssize_t i = 0, l = asdl_seq_LEN(seqs); i < l; i++) {
|
|
asdl_seq *inner_seq = asdl_seq_GET_UNTYPED(seqs, i);
|
|
size += asdl_seq_LEN(inner_seq);
|
|
}
|
|
return size;
|
|
}
|
|
|
|
/* Flattens an asdl_seq* of asdl_seq*s */
|
|
asdl_seq *
|
|
_PyPegen_seq_flatten(Parser *p, asdl_seq *seqs)
|
|
{
|
|
Py_ssize_t flattened_seq_size = _get_flattened_seq_size(seqs);
|
|
assert(flattened_seq_size > 0);
|
|
|
|
asdl_seq *flattened_seq = (asdl_seq*)_Py_asdl_generic_seq_new(flattened_seq_size, p->arena);
|
|
if (!flattened_seq) {
|
|
return NULL;
|
|
}
|
|
|
|
int flattened_seq_idx = 0;
|
|
for (Py_ssize_t i = 0, l = asdl_seq_LEN(seqs); i < l; i++) {
|
|
asdl_seq *inner_seq = asdl_seq_GET_UNTYPED(seqs, i);
|
|
for (Py_ssize_t j = 0, li = asdl_seq_LEN(inner_seq); j < li; j++) {
|
|
asdl_seq_SET_UNTYPED(flattened_seq, flattened_seq_idx++, asdl_seq_GET_UNTYPED(inner_seq, j));
|
|
}
|
|
}
|
|
assert(flattened_seq_idx == flattened_seq_size);
|
|
|
|
return flattened_seq;
|
|
}
|
|
|
|
void *
|
|
_PyPegen_seq_last_item(asdl_seq *seq)
|
|
{
|
|
Py_ssize_t len = asdl_seq_LEN(seq);
|
|
return asdl_seq_GET_UNTYPED(seq, len - 1);
|
|
}
|
|
|
|
/* Creates a new name of the form <first_name>.<second_name> */
|
|
expr_ty
|
|
_PyPegen_join_names_with_dot(Parser *p, expr_ty first_name, expr_ty second_name)
|
|
{
|
|
assert(first_name != NULL && second_name != NULL);
|
|
PyObject *first_identifier = first_name->v.Name.id;
|
|
PyObject *second_identifier = second_name->v.Name.id;
|
|
|
|
if (PyUnicode_READY(first_identifier) == -1) {
|
|
return NULL;
|
|
}
|
|
if (PyUnicode_READY(second_identifier) == -1) {
|
|
return NULL;
|
|
}
|
|
const char *first_str = PyUnicode_AsUTF8(first_identifier);
|
|
if (!first_str) {
|
|
return NULL;
|
|
}
|
|
const char *second_str = PyUnicode_AsUTF8(second_identifier);
|
|
if (!second_str) {
|
|
return NULL;
|
|
}
|
|
Py_ssize_t len = strlen(first_str) + strlen(second_str) + 1; // +1 for the dot
|
|
|
|
PyObject *str = PyBytes_FromStringAndSize(NULL, len);
|
|
if (!str) {
|
|
return NULL;
|
|
}
|
|
|
|
char *s = PyBytes_AS_STRING(str);
|
|
if (!s) {
|
|
return NULL;
|
|
}
|
|
|
|
strcpy(s, first_str);
|
|
s += strlen(first_str);
|
|
*s++ = '.';
|
|
strcpy(s, second_str);
|
|
s += strlen(second_str);
|
|
*s = '\0';
|
|
|
|
PyObject *uni = PyUnicode_DecodeUTF8(PyBytes_AS_STRING(str), PyBytes_GET_SIZE(str), NULL);
|
|
Py_DECREF(str);
|
|
if (!uni) {
|
|
return NULL;
|
|
}
|
|
PyUnicode_InternInPlace(&uni);
|
|
if (_PyArena_AddPyObject(p->arena, uni) < 0) {
|
|
Py_DECREF(uni);
|
|
return NULL;
|
|
}
|
|
|
|
return _PyAST_Name(uni, Load, EXTRA_EXPR(first_name, second_name));
|
|
}
|
|
|
|
/* Counts the total number of dots in seq's tokens */
|
|
int
|
|
_PyPegen_seq_count_dots(asdl_seq *seq)
|
|
{
|
|
int number_of_dots = 0;
|
|
for (Py_ssize_t i = 0, l = asdl_seq_LEN(seq); i < l; i++) {
|
|
Token *current_expr = asdl_seq_GET_UNTYPED(seq, i);
|
|
switch (current_expr->type) {
|
|
case ELLIPSIS:
|
|
number_of_dots += 3;
|
|
break;
|
|
case DOT:
|
|
number_of_dots += 1;
|
|
break;
|
|
default:
|
|
Py_UNREACHABLE();
|
|
}
|
|
}
|
|
|
|
return number_of_dots;
|
|
}
|
|
|
|
/* Creates an alias with '*' as the identifier name */
|
|
alias_ty
|
|
_PyPegen_alias_for_star(Parser *p, int lineno, int col_offset, int end_lineno,
|
|
int end_col_offset, PyArena *arena) {
|
|
PyObject *str = PyUnicode_InternFromString("*");
|
|
if (!str) {
|
|
return NULL;
|
|
}
|
|
if (_PyArena_AddPyObject(p->arena, str) < 0) {
|
|
Py_DECREF(str);
|
|
return NULL;
|
|
}
|
|
return _PyAST_alias(str, NULL, lineno, col_offset, end_lineno, end_col_offset, arena);
|
|
}
|
|
|
|
/* Creates a new asdl_seq* with the identifiers of all the names in seq */
|
|
asdl_identifier_seq *
|
|
_PyPegen_map_names_to_ids(Parser *p, asdl_expr_seq *seq)
|
|
{
|
|
Py_ssize_t len = asdl_seq_LEN(seq);
|
|
assert(len > 0);
|
|
|
|
asdl_identifier_seq *new_seq = _Py_asdl_identifier_seq_new(len, p->arena);
|
|
if (!new_seq) {
|
|
return NULL;
|
|
}
|
|
for (Py_ssize_t i = 0; i < len; i++) {
|
|
expr_ty e = asdl_seq_GET(seq, i);
|
|
asdl_seq_SET(new_seq, i, e->v.Name.id);
|
|
}
|
|
return new_seq;
|
|
}
|
|
|
|
/* Constructs a CmpopExprPair */
|
|
CmpopExprPair *
|
|
_PyPegen_cmpop_expr_pair(Parser *p, cmpop_ty cmpop, expr_ty expr)
|
|
{
|
|
assert(expr != NULL);
|
|
CmpopExprPair *a = _PyArena_Malloc(p->arena, sizeof(CmpopExprPair));
|
|
if (!a) {
|
|
return NULL;
|
|
}
|
|
a->cmpop = cmpop;
|
|
a->expr = expr;
|
|
return a;
|
|
}
|
|
|
|
asdl_int_seq *
|
|
_PyPegen_get_cmpops(Parser *p, asdl_seq *seq)
|
|
{
|
|
Py_ssize_t len = asdl_seq_LEN(seq);
|
|
assert(len > 0);
|
|
|
|
asdl_int_seq *new_seq = _Py_asdl_int_seq_new(len, p->arena);
|
|
if (!new_seq) {
|
|
return NULL;
|
|
}
|
|
for (Py_ssize_t i = 0; i < len; i++) {
|
|
CmpopExprPair *pair = asdl_seq_GET_UNTYPED(seq, i);
|
|
asdl_seq_SET(new_seq, i, pair->cmpop);
|
|
}
|
|
return new_seq;
|
|
}
|
|
|
|
asdl_expr_seq *
|
|
_PyPegen_get_exprs(Parser *p, asdl_seq *seq)
|
|
{
|
|
Py_ssize_t len = asdl_seq_LEN(seq);
|
|
assert(len > 0);
|
|
|
|
asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(len, p->arena);
|
|
if (!new_seq) {
|
|
return NULL;
|
|
}
|
|
for (Py_ssize_t i = 0; i < len; i++) {
|
|
CmpopExprPair *pair = asdl_seq_GET_UNTYPED(seq, i);
|
|
asdl_seq_SET(new_seq, i, pair->expr);
|
|
}
|
|
return new_seq;
|
|
}
|
|
|
|
/* Creates an asdl_seq* where all the elements have been changed to have ctx as context */
|
|
static asdl_expr_seq *
|
|
_set_seq_context(Parser *p, asdl_expr_seq *seq, expr_context_ty ctx)
|
|
{
|
|
Py_ssize_t len = asdl_seq_LEN(seq);
|
|
if (len == 0) {
|
|
return NULL;
|
|
}
|
|
|
|
asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(len, p->arena);
|
|
if (!new_seq) {
|
|
return NULL;
|
|
}
|
|
for (Py_ssize_t i = 0; i < len; i++) {
|
|
expr_ty e = asdl_seq_GET(seq, i);
|
|
asdl_seq_SET(new_seq, i, _PyPegen_set_expr_context(p, e, ctx));
|
|
}
|
|
return new_seq;
|
|
}
|
|
|
|
static expr_ty
|
|
_set_name_context(Parser *p, expr_ty e, expr_context_ty ctx)
|
|
{
|
|
return _PyAST_Name(e->v.Name.id, ctx, EXTRA_EXPR(e, e));
|
|
}
|
|
|
|
static expr_ty
|
|
_set_tuple_context(Parser *p, expr_ty e, expr_context_ty ctx)
|
|
{
|
|
return _PyAST_Tuple(
|
|
_set_seq_context(p, e->v.Tuple.elts, ctx),
|
|
ctx,
|
|
EXTRA_EXPR(e, e));
|
|
}
|
|
|
|
static expr_ty
|
|
_set_list_context(Parser *p, expr_ty e, expr_context_ty ctx)
|
|
{
|
|
return _PyAST_List(
|
|
_set_seq_context(p, e->v.List.elts, ctx),
|
|
ctx,
|
|
EXTRA_EXPR(e, e));
|
|
}
|
|
|
|
static expr_ty
|
|
_set_subscript_context(Parser *p, expr_ty e, expr_context_ty ctx)
|
|
{
|
|
return _PyAST_Subscript(e->v.Subscript.value, e->v.Subscript.slice,
|
|
ctx, EXTRA_EXPR(e, e));
|
|
}
|
|
|
|
static expr_ty
|
|
_set_attribute_context(Parser *p, expr_ty e, expr_context_ty ctx)
|
|
{
|
|
return _PyAST_Attribute(e->v.Attribute.value, e->v.Attribute.attr,
|
|
ctx, EXTRA_EXPR(e, e));
|
|
}
|
|
|
|
static expr_ty
|
|
_set_starred_context(Parser *p, expr_ty e, expr_context_ty ctx)
|
|
{
|
|
return _PyAST_Starred(_PyPegen_set_expr_context(p, e->v.Starred.value, ctx),
|
|
ctx, EXTRA_EXPR(e, e));
|
|
}
|
|
|
|
/* Creates an `expr_ty` equivalent to `expr` but with `ctx` as context */
|
|
expr_ty
|
|
_PyPegen_set_expr_context(Parser *p, expr_ty expr, expr_context_ty ctx)
|
|
{
|
|
assert(expr != NULL);
|
|
|
|
expr_ty new = NULL;
|
|
switch (expr->kind) {
|
|
case Name_kind:
|
|
new = _set_name_context(p, expr, ctx);
|
|
break;
|
|
case Tuple_kind:
|
|
new = _set_tuple_context(p, expr, ctx);
|
|
break;
|
|
case List_kind:
|
|
new = _set_list_context(p, expr, ctx);
|
|
break;
|
|
case Subscript_kind:
|
|
new = _set_subscript_context(p, expr, ctx);
|
|
break;
|
|
case Attribute_kind:
|
|
new = _set_attribute_context(p, expr, ctx);
|
|
break;
|
|
case Starred_kind:
|
|
new = _set_starred_context(p, expr, ctx);
|
|
break;
|
|
default:
|
|
new = expr;
|
|
}
|
|
return new;
|
|
}
|
|
|
|
/* Constructs a KeyValuePair that is used when parsing a dict's key value pairs */
|
|
KeyValuePair *
|
|
_PyPegen_key_value_pair(Parser *p, expr_ty key, expr_ty value)
|
|
{
|
|
KeyValuePair *a = _PyArena_Malloc(p->arena, sizeof(KeyValuePair));
|
|
if (!a) {
|
|
return NULL;
|
|
}
|
|
a->key = key;
|
|
a->value = value;
|
|
return a;
|
|
}
|
|
|
|
/* Extracts all keys from an asdl_seq* of KeyValuePair*'s */
|
|
asdl_expr_seq *
|
|
_PyPegen_get_keys(Parser *p, asdl_seq *seq)
|
|
{
|
|
Py_ssize_t len = asdl_seq_LEN(seq);
|
|
asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(len, p->arena);
|
|
if (!new_seq) {
|
|
return NULL;
|
|
}
|
|
for (Py_ssize_t i = 0; i < len; i++) {
|
|
KeyValuePair *pair = asdl_seq_GET_UNTYPED(seq, i);
|
|
asdl_seq_SET(new_seq, i, pair->key);
|
|
}
|
|
return new_seq;
|
|
}
|
|
|
|
/* Extracts all values from an asdl_seq* of KeyValuePair*'s */
|
|
asdl_expr_seq *
|
|
_PyPegen_get_values(Parser *p, asdl_seq *seq)
|
|
{
|
|
Py_ssize_t len = asdl_seq_LEN(seq);
|
|
asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(len, p->arena);
|
|
if (!new_seq) {
|
|
return NULL;
|
|
}
|
|
for (Py_ssize_t i = 0; i < len; i++) {
|
|
KeyValuePair *pair = asdl_seq_GET_UNTYPED(seq, i);
|
|
asdl_seq_SET(new_seq, i, pair->value);
|
|
}
|
|
return new_seq;
|
|
}
|
|
|
|
/* Constructs a NameDefaultPair */
|
|
NameDefaultPair *
|
|
_PyPegen_name_default_pair(Parser *p, arg_ty arg, expr_ty value, Token *tc)
|
|
{
|
|
NameDefaultPair *a = _PyArena_Malloc(p->arena, sizeof(NameDefaultPair));
|
|
if (!a) {
|
|
return NULL;
|
|
}
|
|
a->arg = _PyPegen_add_type_comment_to_arg(p, arg, tc);
|
|
a->value = value;
|
|
return a;
|
|
}
|
|
|
|
/* Constructs a SlashWithDefault */
|
|
SlashWithDefault *
|
|
_PyPegen_slash_with_default(Parser *p, asdl_arg_seq *plain_names, asdl_seq *names_with_defaults)
|
|
{
|
|
SlashWithDefault *a = _PyArena_Malloc(p->arena, sizeof(SlashWithDefault));
|
|
if (!a) {
|
|
return NULL;
|
|
}
|
|
a->plain_names = plain_names;
|
|
a->names_with_defaults = names_with_defaults;
|
|
return a;
|
|
}
|
|
|
|
/* Constructs a StarEtc */
|
|
StarEtc *
|
|
_PyPegen_star_etc(Parser *p, arg_ty vararg, asdl_seq *kwonlyargs, arg_ty kwarg)
|
|
{
|
|
StarEtc *a = _PyArena_Malloc(p->arena, sizeof(StarEtc));
|
|
if (!a) {
|
|
return NULL;
|
|
}
|
|
a->vararg = vararg;
|
|
a->kwonlyargs = kwonlyargs;
|
|
a->kwarg = kwarg;
|
|
return a;
|
|
}
|
|
|
|
asdl_seq *
|
|
_PyPegen_join_sequences(Parser *p, asdl_seq *a, asdl_seq *b)
|
|
{
|
|
Py_ssize_t first_len = asdl_seq_LEN(a);
|
|
Py_ssize_t second_len = asdl_seq_LEN(b);
|
|
asdl_seq *new_seq = (asdl_seq*)_Py_asdl_generic_seq_new(first_len + second_len, p->arena);
|
|
if (!new_seq) {
|
|
return NULL;
|
|
}
|
|
|
|
int k = 0;
|
|
for (Py_ssize_t i = 0; i < first_len; i++) {
|
|
asdl_seq_SET_UNTYPED(new_seq, k++, asdl_seq_GET_UNTYPED(a, i));
|
|
}
|
|
for (Py_ssize_t i = 0; i < second_len; i++) {
|
|
asdl_seq_SET_UNTYPED(new_seq, k++, asdl_seq_GET_UNTYPED(b, i));
|
|
}
|
|
|
|
return new_seq;
|
|
}
|
|
|
|
static asdl_arg_seq*
|
|
_get_names(Parser *p, asdl_seq *names_with_defaults)
|
|
{
|
|
Py_ssize_t len = asdl_seq_LEN(names_with_defaults);
|
|
asdl_arg_seq *seq = _Py_asdl_arg_seq_new(len, p->arena);
|
|
if (!seq) {
|
|
return NULL;
|
|
}
|
|
for (Py_ssize_t i = 0; i < len; i++) {
|
|
NameDefaultPair *pair = asdl_seq_GET_UNTYPED(names_with_defaults, i);
|
|
asdl_seq_SET(seq, i, pair->arg);
|
|
}
|
|
return seq;
|
|
}
|
|
|
|
static asdl_expr_seq *
|
|
_get_defaults(Parser *p, asdl_seq *names_with_defaults)
|
|
{
|
|
Py_ssize_t len = asdl_seq_LEN(names_with_defaults);
|
|
asdl_expr_seq *seq = _Py_asdl_expr_seq_new(len, p->arena);
|
|
if (!seq) {
|
|
return NULL;
|
|
}
|
|
for (Py_ssize_t i = 0; i < len; i++) {
|
|
NameDefaultPair *pair = asdl_seq_GET_UNTYPED(names_with_defaults, i);
|
|
asdl_seq_SET(seq, i, pair->value);
|
|
}
|
|
return seq;
|
|
}
|
|
|
|
static int
|
|
_make_posonlyargs(Parser *p,
|
|
asdl_arg_seq *slash_without_default,
|
|
SlashWithDefault *slash_with_default,
|
|
asdl_arg_seq **posonlyargs) {
|
|
if (slash_without_default != NULL) {
|
|
*posonlyargs = slash_without_default;
|
|
}
|
|
else if (slash_with_default != NULL) {
|
|
asdl_arg_seq *slash_with_default_names =
|
|
_get_names(p, slash_with_default->names_with_defaults);
|
|
if (!slash_with_default_names) {
|
|
return -1;
|
|
}
|
|
*posonlyargs = (asdl_arg_seq*)_PyPegen_join_sequences(
|
|
p,
|
|
(asdl_seq*)slash_with_default->plain_names,
|
|
(asdl_seq*)slash_with_default_names);
|
|
}
|
|
else {
|
|
*posonlyargs = _Py_asdl_arg_seq_new(0, p->arena);
|
|
}
|
|
return *posonlyargs == NULL ? -1 : 0;
|
|
}
|
|
|
|
static int
|
|
_make_posargs(Parser *p,
|
|
asdl_arg_seq *plain_names,
|
|
asdl_seq *names_with_default,
|
|
asdl_arg_seq **posargs) {
|
|
if (plain_names != NULL && names_with_default != NULL) {
|
|
asdl_arg_seq *names_with_default_names = _get_names(p, names_with_default);
|
|
if (!names_with_default_names) {
|
|
return -1;
|
|
}
|
|
*posargs = (asdl_arg_seq*)_PyPegen_join_sequences(
|
|
p,(asdl_seq*)plain_names, (asdl_seq*)names_with_default_names);
|
|
}
|
|
else if (plain_names == NULL && names_with_default != NULL) {
|
|
*posargs = _get_names(p, names_with_default);
|
|
}
|
|
else if (plain_names != NULL && names_with_default == NULL) {
|
|
*posargs = plain_names;
|
|
}
|
|
else {
|
|
*posargs = _Py_asdl_arg_seq_new(0, p->arena);
|
|
}
|
|
return *posargs == NULL ? -1 : 0;
|
|
}
|
|
|
|
static int
|
|
_make_posdefaults(Parser *p,
|
|
SlashWithDefault *slash_with_default,
|
|
asdl_seq *names_with_default,
|
|
asdl_expr_seq **posdefaults) {
|
|
if (slash_with_default != NULL && names_with_default != NULL) {
|
|
asdl_expr_seq *slash_with_default_values =
|
|
_get_defaults(p, slash_with_default->names_with_defaults);
|
|
if (!slash_with_default_values) {
|
|
return -1;
|
|
}
|
|
asdl_expr_seq *names_with_default_values = _get_defaults(p, names_with_default);
|
|
if (!names_with_default_values) {
|
|
return -1;
|
|
}
|
|
*posdefaults = (asdl_expr_seq*)_PyPegen_join_sequences(
|
|
p,
|
|
(asdl_seq*)slash_with_default_values,
|
|
(asdl_seq*)names_with_default_values);
|
|
}
|
|
else if (slash_with_default == NULL && names_with_default != NULL) {
|
|
*posdefaults = _get_defaults(p, names_with_default);
|
|
}
|
|
else if (slash_with_default != NULL && names_with_default == NULL) {
|
|
*posdefaults = _get_defaults(p, slash_with_default->names_with_defaults);
|
|
}
|
|
else {
|
|
*posdefaults = _Py_asdl_expr_seq_new(0, p->arena);
|
|
}
|
|
return *posdefaults == NULL ? -1 : 0;
|
|
}
|
|
|
|
static int
|
|
_make_kwargs(Parser *p, StarEtc *star_etc,
|
|
asdl_arg_seq **kwonlyargs,
|
|
asdl_expr_seq **kwdefaults) {
|
|
if (star_etc != NULL && star_etc->kwonlyargs != NULL) {
|
|
*kwonlyargs = _get_names(p, star_etc->kwonlyargs);
|
|
}
|
|
else {
|
|
*kwonlyargs = _Py_asdl_arg_seq_new(0, p->arena);
|
|
}
|
|
|
|
if (*kwonlyargs == NULL) {
|
|
return -1;
|
|
}
|
|
|
|
if (star_etc != NULL && star_etc->kwonlyargs != NULL) {
|
|
*kwdefaults = _get_defaults(p, star_etc->kwonlyargs);
|
|
}
|
|
else {
|
|
*kwdefaults = _Py_asdl_expr_seq_new(0, p->arena);
|
|
}
|
|
|
|
if (*kwdefaults == NULL) {
|
|
return -1;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* Constructs an arguments_ty object out of all the parsed constructs in the parameters rule */
|
|
arguments_ty
|
|
_PyPegen_make_arguments(Parser *p, asdl_arg_seq *slash_without_default,
|
|
SlashWithDefault *slash_with_default, asdl_arg_seq *plain_names,
|
|
asdl_seq *names_with_default, StarEtc *star_etc)
|
|
{
|
|
asdl_arg_seq *posonlyargs;
|
|
if (_make_posonlyargs(p, slash_without_default, slash_with_default, &posonlyargs) == -1) {
|
|
return NULL;
|
|
}
|
|
|
|
asdl_arg_seq *posargs;
|
|
if (_make_posargs(p, plain_names, names_with_default, &posargs) == -1) {
|
|
return NULL;
|
|
}
|
|
|
|
asdl_expr_seq *posdefaults;
|
|
if (_make_posdefaults(p,slash_with_default, names_with_default, &posdefaults) == -1) {
|
|
return NULL;
|
|
}
|
|
|
|
arg_ty vararg = NULL;
|
|
if (star_etc != NULL && star_etc->vararg != NULL) {
|
|
vararg = star_etc->vararg;
|
|
}
|
|
|
|
asdl_arg_seq *kwonlyargs;
|
|
asdl_expr_seq *kwdefaults;
|
|
if (_make_kwargs(p, star_etc, &kwonlyargs, &kwdefaults) == -1) {
|
|
return NULL;
|
|
}
|
|
|
|
arg_ty kwarg = NULL;
|
|
if (star_etc != NULL && star_etc->kwarg != NULL) {
|
|
kwarg = star_etc->kwarg;
|
|
}
|
|
|
|
return _PyAST_arguments(posonlyargs, posargs, vararg, kwonlyargs,
|
|
kwdefaults, kwarg, posdefaults, p->arena);
|
|
}
|
|
|
|
|
|
/* Constructs an empty arguments_ty object, that gets used when a function accepts no
|
|
* arguments. */
|
|
arguments_ty
|
|
_PyPegen_empty_arguments(Parser *p)
|
|
{
|
|
asdl_arg_seq *posonlyargs = _Py_asdl_arg_seq_new(0, p->arena);
|
|
if (!posonlyargs) {
|
|
return NULL;
|
|
}
|
|
asdl_arg_seq *posargs = _Py_asdl_arg_seq_new(0, p->arena);
|
|
if (!posargs) {
|
|
return NULL;
|
|
}
|
|
asdl_expr_seq *posdefaults = _Py_asdl_expr_seq_new(0, p->arena);
|
|
if (!posdefaults) {
|
|
return NULL;
|
|
}
|
|
asdl_arg_seq *kwonlyargs = _Py_asdl_arg_seq_new(0, p->arena);
|
|
if (!kwonlyargs) {
|
|
return NULL;
|
|
}
|
|
asdl_expr_seq *kwdefaults = _Py_asdl_expr_seq_new(0, p->arena);
|
|
if (!kwdefaults) {
|
|
return NULL;
|
|
}
|
|
|
|
return _PyAST_arguments(posonlyargs, posargs, NULL, kwonlyargs,
|
|
kwdefaults, NULL, posdefaults, p->arena);
|
|
}
|
|
|
|
/* Encapsulates the value of an operator_ty into an AugOperator struct */
|
|
AugOperator *
|
|
_PyPegen_augoperator(Parser *p, operator_ty kind)
|
|
{
|
|
AugOperator *a = _PyArena_Malloc(p->arena, sizeof(AugOperator));
|
|
if (!a) {
|
|
return NULL;
|
|
}
|
|
a->kind = kind;
|
|
return a;
|
|
}
|
|
|
|
/* Construct a FunctionDef equivalent to function_def, but with decorators */
|
|
stmt_ty
|
|
_PyPegen_function_def_decorators(Parser *p, asdl_expr_seq *decorators, stmt_ty function_def)
|
|
{
|
|
assert(function_def != NULL);
|
|
if (function_def->kind == AsyncFunctionDef_kind) {
|
|
return _PyAST_AsyncFunctionDef(
|
|
function_def->v.FunctionDef.name, function_def->v.FunctionDef.args,
|
|
function_def->v.FunctionDef.body, decorators, function_def->v.FunctionDef.returns,
|
|
function_def->v.FunctionDef.type_comment, function_def->lineno,
|
|
function_def->col_offset, function_def->end_lineno, function_def->end_col_offset,
|
|
p->arena);
|
|
}
|
|
|
|
return _PyAST_FunctionDef(
|
|
function_def->v.FunctionDef.name, function_def->v.FunctionDef.args,
|
|
function_def->v.FunctionDef.body, decorators,
|
|
function_def->v.FunctionDef.returns,
|
|
function_def->v.FunctionDef.type_comment, function_def->lineno,
|
|
function_def->col_offset, function_def->end_lineno,
|
|
function_def->end_col_offset, p->arena);
|
|
}
|
|
|
|
/* Construct a ClassDef equivalent to class_def, but with decorators */
|
|
stmt_ty
|
|
_PyPegen_class_def_decorators(Parser *p, asdl_expr_seq *decorators, stmt_ty class_def)
|
|
{
|
|
assert(class_def != NULL);
|
|
return _PyAST_ClassDef(
|
|
class_def->v.ClassDef.name, class_def->v.ClassDef.bases,
|
|
class_def->v.ClassDef.keywords, class_def->v.ClassDef.body, decorators,
|
|
class_def->lineno, class_def->col_offset, class_def->end_lineno,
|
|
class_def->end_col_offset, p->arena);
|
|
}
|
|
|
|
/* Construct a KeywordOrStarred */
|
|
KeywordOrStarred *
|
|
_PyPegen_keyword_or_starred(Parser *p, void *element, int is_keyword)
|
|
{
|
|
KeywordOrStarred *a = _PyArena_Malloc(p->arena, sizeof(KeywordOrStarred));
|
|
if (!a) {
|
|
return NULL;
|
|
}
|
|
a->element = element;
|
|
a->is_keyword = is_keyword;
|
|
return a;
|
|
}
|
|
|
|
/* Get the number of starred expressions in an asdl_seq* of KeywordOrStarred*s */
|
|
static int
|
|
_seq_number_of_starred_exprs(asdl_seq *seq)
|
|
{
|
|
int n = 0;
|
|
for (Py_ssize_t i = 0, l = asdl_seq_LEN(seq); i < l; i++) {
|
|
KeywordOrStarred *k = asdl_seq_GET_UNTYPED(seq, i);
|
|
if (!k->is_keyword) {
|
|
n++;
|
|
}
|
|
}
|
|
return n;
|
|
}
|
|
|
|
/* Extract the starred expressions of an asdl_seq* of KeywordOrStarred*s */
|
|
asdl_expr_seq *
|
|
_PyPegen_seq_extract_starred_exprs(Parser *p, asdl_seq *kwargs)
|
|
{
|
|
int new_len = _seq_number_of_starred_exprs(kwargs);
|
|
if (new_len == 0) {
|
|
return NULL;
|
|
}
|
|
asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(new_len, p->arena);
|
|
if (!new_seq) {
|
|
return NULL;
|
|
}
|
|
|
|
int idx = 0;
|
|
for (Py_ssize_t i = 0, len = asdl_seq_LEN(kwargs); i < len; i++) {
|
|
KeywordOrStarred *k = asdl_seq_GET_UNTYPED(kwargs, i);
|
|
if (!k->is_keyword) {
|
|
asdl_seq_SET(new_seq, idx++, k->element);
|
|
}
|
|
}
|
|
return new_seq;
|
|
}
|
|
|
|
/* Return a new asdl_seq* with only the keywords in kwargs */
|
|
asdl_keyword_seq*
|
|
_PyPegen_seq_delete_starred_exprs(Parser *p, asdl_seq *kwargs)
|
|
{
|
|
Py_ssize_t len = asdl_seq_LEN(kwargs);
|
|
Py_ssize_t new_len = len - _seq_number_of_starred_exprs(kwargs);
|
|
if (new_len == 0) {
|
|
return NULL;
|
|
}
|
|
asdl_keyword_seq *new_seq = _Py_asdl_keyword_seq_new(new_len, p->arena);
|
|
if (!new_seq) {
|
|
return NULL;
|
|
}
|
|
|
|
int idx = 0;
|
|
for (Py_ssize_t i = 0; i < len; i++) {
|
|
KeywordOrStarred *k = asdl_seq_GET_UNTYPED(kwargs, i);
|
|
if (k->is_keyword) {
|
|
asdl_seq_SET(new_seq, idx++, k->element);
|
|
}
|
|
}
|
|
return new_seq;
|
|
}
|
|
|
|
expr_ty
|
|
_PyPegen_concatenate_strings(Parser *p, asdl_seq *strings)
|
|
{
|
|
Py_ssize_t len = asdl_seq_LEN(strings);
|
|
assert(len > 0);
|
|
|
|
Token *first = asdl_seq_GET_UNTYPED(strings, 0);
|
|
Token *last = asdl_seq_GET_UNTYPED(strings, len - 1);
|
|
|
|
int bytesmode = 0;
|
|
PyObject *bytes_str = NULL;
|
|
|
|
FstringParser state;
|
|
_PyPegen_FstringParser_Init(&state);
|
|
|
|
for (Py_ssize_t i = 0; i < len; i++) {
|
|
Token *t = asdl_seq_GET_UNTYPED(strings, i);
|
|
|
|
int this_bytesmode;
|
|
int this_rawmode;
|
|
PyObject *s;
|
|
const char *fstr;
|
|
Py_ssize_t fstrlen = -1;
|
|
|
|
if (_PyPegen_parsestr(p, &this_bytesmode, &this_rawmode, &s, &fstr, &fstrlen, t) != 0) {
|
|
goto error;
|
|
}
|
|
|
|
/* Check that we are not mixing bytes with unicode. */
|
|
if (i != 0 && bytesmode != this_bytesmode) {
|
|
RAISE_SYNTAX_ERROR("cannot mix bytes and nonbytes literals");
|
|
Py_XDECREF(s);
|
|
goto error;
|
|
}
|
|
bytesmode = this_bytesmode;
|
|
|
|
if (fstr != NULL) {
|
|
assert(s == NULL && !bytesmode);
|
|
|
|
int result = _PyPegen_FstringParser_ConcatFstring(p, &state, &fstr, fstr + fstrlen,
|
|
this_rawmode, 0, first, t, last);
|
|
if (result < 0) {
|
|
goto error;
|
|
}
|
|
}
|
|
else {
|
|
/* String or byte string. */
|
|
assert(s != NULL && fstr == NULL);
|
|
assert(bytesmode ? PyBytes_CheckExact(s) : PyUnicode_CheckExact(s));
|
|
|
|
if (bytesmode) {
|
|
if (i == 0) {
|
|
bytes_str = s;
|
|
}
|
|
else {
|
|
PyBytes_ConcatAndDel(&bytes_str, s);
|
|
if (!bytes_str) {
|
|
goto error;
|
|
}
|
|
}
|
|
}
|
|
else {
|
|
/* This is a regular string. Concatenate it. */
|
|
if (_PyPegen_FstringParser_ConcatAndDel(&state, s) < 0) {
|
|
goto error;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (bytesmode) {
|
|
if (_PyArena_AddPyObject(p->arena, bytes_str) < 0) {
|
|
goto error;
|
|
}
|
|
return _PyAST_Constant(bytes_str, NULL, first->lineno,
|
|
first->col_offset, last->end_lineno,
|
|
last->end_col_offset, p->arena);
|
|
}
|
|
|
|
return _PyPegen_FstringParser_Finish(p, &state, first, last);
|
|
|
|
error:
|
|
Py_XDECREF(bytes_str);
|
|
_PyPegen_FstringParser_Dealloc(&state);
|
|
if (PyErr_Occurred()) {
|
|
raise_decode_error(p);
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
mod_ty
|
|
_PyPegen_make_module(Parser *p, asdl_stmt_seq *a) {
|
|
asdl_type_ignore_seq *type_ignores = NULL;
|
|
Py_ssize_t num = p->type_ignore_comments.num_items;
|
|
if (num > 0) {
|
|
// Turn the raw (comment, lineno) pairs into TypeIgnore objects in the arena
|
|
type_ignores = _Py_asdl_type_ignore_seq_new(num, p->arena);
|
|
if (type_ignores == NULL) {
|
|
return NULL;
|
|
}
|
|
for (int i = 0; i < num; i++) {
|
|
PyObject *tag = _PyPegen_new_type_comment(p, p->type_ignore_comments.items[i].comment);
|
|
if (tag == NULL) {
|
|
return NULL;
|
|
}
|
|
type_ignore_ty ti = _PyAST_TypeIgnore(p->type_ignore_comments.items[i].lineno,
|
|
tag, p->arena);
|
|
if (ti == NULL) {
|
|
return NULL;
|
|
}
|
|
asdl_seq_SET(type_ignores, i, ti);
|
|
}
|
|
}
|
|
return _PyAST_Module(a, type_ignores, p->arena);
|
|
}
|
|
|
|
// Error reporting helpers
|
|
|
|
expr_ty
|
|
_PyPegen_get_invalid_target(expr_ty e, TARGETS_TYPE targets_type)
|
|
{
|
|
if (e == NULL) {
|
|
return NULL;
|
|
}
|
|
|
|
#define VISIT_CONTAINER(CONTAINER, TYPE) do { \
|
|
Py_ssize_t len = asdl_seq_LEN((CONTAINER)->v.TYPE.elts);\
|
|
for (Py_ssize_t i = 0; i < len; i++) {\
|
|
expr_ty other = asdl_seq_GET((CONTAINER)->v.TYPE.elts, i);\
|
|
expr_ty child = _PyPegen_get_invalid_target(other, targets_type);\
|
|
if (child != NULL) {\
|
|
return child;\
|
|
}\
|
|
}\
|
|
} while (0)
|
|
|
|
// We only need to visit List and Tuple nodes recursively as those
|
|
// are the only ones that can contain valid names in targets when
|
|
// they are parsed as expressions. Any other kind of expression
|
|
// that is a container (like Sets or Dicts) is directly invalid and
|
|
// we don't need to visit it recursively.
|
|
|
|
switch (e->kind) {
|
|
case List_kind:
|
|
VISIT_CONTAINER(e, List);
|
|
return NULL;
|
|
case Tuple_kind:
|
|
VISIT_CONTAINER(e, Tuple);
|
|
return NULL;
|
|
case Starred_kind:
|
|
if (targets_type == DEL_TARGETS) {
|
|
return e;
|
|
}
|
|
return _PyPegen_get_invalid_target(e->v.Starred.value, targets_type);
|
|
case Compare_kind:
|
|
// This is needed, because the `a in b` in `for a in b` gets parsed
|
|
// as a comparison, and so we need to search the left side of the comparison
|
|
// for invalid targets.
|
|
if (targets_type == FOR_TARGETS) {
|
|
cmpop_ty cmpop = (cmpop_ty) asdl_seq_GET(e->v.Compare.ops, 0);
|
|
if (cmpop == In) {
|
|
return _PyPegen_get_invalid_target(e->v.Compare.left, targets_type);
|
|
}
|
|
return NULL;
|
|
}
|
|
return e;
|
|
case Name_kind:
|
|
case Subscript_kind:
|
|
case Attribute_kind:
|
|
return NULL;
|
|
default:
|
|
return e;
|
|
}
|
|
}
|
|
|
|
void *_PyPegen_arguments_parsing_error(Parser *p, expr_ty e) {
|
|
int kwarg_unpacking = 0;
|
|
for (Py_ssize_t i = 0, l = asdl_seq_LEN(e->v.Call.keywords); i < l; i++) {
|
|
keyword_ty keyword = asdl_seq_GET(e->v.Call.keywords, i);
|
|
if (!keyword->arg) {
|
|
kwarg_unpacking = 1;
|
|
}
|
|
}
|
|
|
|
const char *msg = NULL;
|
|
if (kwarg_unpacking) {
|
|
msg = "positional argument follows keyword argument unpacking";
|
|
} else {
|
|
msg = "positional argument follows keyword argument";
|
|
}
|
|
|
|
return RAISE_SYNTAX_ERROR(msg);
|
|
}
|
|
|
|
void *
|
|
_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args)
|
|
{
|
|
/* The rule that calls this function is 'args for_if_clauses'.
|
|
For the input f(L, x for x in y), L and x are in args and
|
|
the for is parsed as a for_if_clause. We have to check if
|
|
len <= 1, so that input like dict((a, b) for a, b in x)
|
|
gets successfully parsed and then we pass the last
|
|
argument (x in the above example) as the location of the
|
|
error */
|
|
Py_ssize_t len = asdl_seq_LEN(args->v.Call.args);
|
|
if (len <= 1) {
|
|
return NULL;
|
|
}
|
|
|
|
return RAISE_SYNTAX_ERROR_STARTING_FROM(
|
|
(expr_ty) asdl_seq_GET(args->v.Call.args, len - 1),
|
|
"Generator expression must be parenthesized"
|
|
);
|
|
}
|
|
|
|
|
|
expr_ty _PyPegen_collect_call_seqs(Parser *p, asdl_expr_seq *a, asdl_seq *b,
|
|
int lineno, int col_offset, int end_lineno,
|
|
int end_col_offset, PyArena *arena) {
|
|
Py_ssize_t args_len = asdl_seq_LEN(a);
|
|
Py_ssize_t total_len = args_len;
|
|
|
|
if (b == NULL) {
|
|
return _PyAST_Call(_PyPegen_dummy_name(p), a, NULL, lineno, col_offset,
|
|
end_lineno, end_col_offset, arena);
|
|
|
|
}
|
|
|
|
asdl_expr_seq *starreds = _PyPegen_seq_extract_starred_exprs(p, b);
|
|
asdl_keyword_seq *keywords = _PyPegen_seq_delete_starred_exprs(p, b);
|
|
|
|
if (starreds) {
|
|
total_len += asdl_seq_LEN(starreds);
|
|
}
|
|
|
|
asdl_expr_seq *args = _Py_asdl_expr_seq_new(total_len, arena);
|
|
|
|
Py_ssize_t i = 0;
|
|
for (i = 0; i < args_len; i++) {
|
|
asdl_seq_SET(args, i, asdl_seq_GET(a, i));
|
|
}
|
|
for (; i < total_len; i++) {
|
|
asdl_seq_SET(args, i, asdl_seq_GET(starreds, i - args_len));
|
|
}
|
|
|
|
return _PyAST_Call(_PyPegen_dummy_name(p), args, keywords, lineno,
|
|
col_offset, end_lineno, end_col_offset, arena);
|
|
}
|