From ad106c68eb00f5e4af2f937107baff6141948cee Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Mon, 3 May 2021 10:43:00 +0300 Subject: [PATCH] bpo-42725: Render annotations effectless on symbol table with PEP 563 (GH-25583) --- Include/internal/pycore_symtable.h | 2 +- Lib/test/test_future.py | 61 +++++++++-- .../2021-04-25-05-40-51.bpo-42725.WGloYm.rst | 2 + Modules/symtablemodule.c | 1 - Python/symtable.c | 101 ++++++++++++++++-- 5 files changed, 149 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-04-25-05-40-51.bpo-42725.WGloYm.rst diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index d6d90f6c26e..f3505f8949b 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -10,7 +10,7 @@ extern "C" { struct _mod; // Type defined in pycore_ast.h -typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock } +typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock, AnnotationBlock } _Py_block_ty; struct _symtable_entry; diff --git a/Lib/test/test_future.py b/Lib/test/test_future.py index 8a09853d1f7..4ef11232a33 100644 --- a/Lib/test/test_future.py +++ b/Lib/test/test_future.py @@ -171,6 +171,14 @@ def assertAnnotationEqual( self.assertEqual(actual, expected) + def _exec_future(self, code): + scope = {} + exec( + "from __future__ import annotations\n" + + code, {}, scope + ) + return scope + def test_annotations(self): eq = self.assertAnnotationEqual eq('...') @@ -310,10 +318,6 @@ def test_annotations(self): eq("f'{x}'") eq("f'{x!r}'") eq("f'{x!a}'") - eq('(yield from outside_of_generator)') - eq('(yield)') - eq('(yield a + b)') - eq('await some.complicated[0].call(with_args=True or 1 is not 1)') eq('[x for x in (a if b else c)]') eq('[x for x in a if (b if c else d)]') eq('f(x for x in a)') @@ -321,8 +325,6 @@ def test_annotations(self): eq('f((x for x in a), 2)') eq('(((a)))', 'a') eq('(((a, b)))', '(a, b)') - eq("(x := 10)") - eq("f'{(x := 10):=10}'") eq("1 + 2 + 3") def test_fstring_debug_annotations(self): @@ -354,6 +356,53 @@ def test_annotation_with_complex_target(self): "object.__debug__: int" ) + def test_annotations_symbol_table_pass(self): + namespace = self._exec_future(dedent(""" + from __future__ import annotations + + def foo(): + outer = 1 + def bar(): + inner: outer = 1 + return bar + """)) + + foo = namespace.pop("foo") + self.assertIsNone(foo().__closure__) + self.assertEqual(foo.__code__.co_cellvars, ()) + self.assertEqual(foo().__code__.co_freevars, ()) + + def test_annotations_forbidden(self): + with self.assertRaises(SyntaxError): + self._exec_future("test: (yield)") + + with self.assertRaises(SyntaxError): + self._exec_future("test.test: (yield a + b)") + + with self.assertRaises(SyntaxError): + self._exec_future("test[something]: (yield from x)") + + with self.assertRaises(SyntaxError): + self._exec_future("def func(test: (yield from outside_of_generator)): pass") + + with self.assertRaises(SyntaxError): + self._exec_future("def test() -> (await y): pass") + + with self.assertRaises(SyntaxError): + self._exec_future("async def test() -> something((a := b)): pass") + + with self.assertRaises(SyntaxError): + self._exec_future("test: await some.complicated[0].call(with_args=True or 1 is not 1)") + + with self.assertRaises(SyntaxError): + self._exec_future("test: f'{(x := 10):=10}'") + + with self.assertRaises(SyntaxError): + self._exec_future(dedent("""\ + def foo(): + def bar(arg: (yield)): pass + """)) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-04-25-05-40-51.bpo-42725.WGloYm.rst b/Misc/NEWS.d/next/Core and Builtins/2021-04-25-05-40-51.bpo-42725.WGloYm.rst new file mode 100644 index 00000000000..c9ea706e982 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-04-25-05-40-51.bpo-42725.WGloYm.rst @@ -0,0 +1,2 @@ +Usage of ``await``/``yield``/``yield from`` and named expressions within an +annotation is now forbidden when PEP 563 is activated. diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index cf10b4deaf4..c25ecc2b5dc 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -58,7 +58,6 @@ _symtable_symtable_impl(PyObject *module, PyObject *source, } t = (PyObject *)st->st_top; Py_INCREF(t); - PyMem_Free((void *)st->st_future); _PySymtable_Free(st); return t; } diff --git a/Python/symtable.c b/Python/symtable.c index e620f1ecc1b..62bd1e2ec48 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1,6 +1,6 @@ #include "Python.h" #include "pycore_ast.h" // identifier, stmt_ty -#include "pycore_compile.h" // _Py_Mangle() +#include "pycore_compile.h" // _Py_Mangle(), _PyFuture_FromAST() #include "pycore_parser.h" // _PyParser_ASTFromString() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_symtable.h" // PySTEntryObject @@ -45,6 +45,10 @@ #define NAMED_EXPR_COMP_ITER_EXPR \ "assignment expression cannot be used in a comprehension iterable expression" +#define ANNOTATION_NOT_ALLOWED \ +"'%s' can not be used within an annotation" + + static PySTEntryObject * ste_new(struct symtable *st, identifier name, _Py_block_ty block, void *key, int lineno, int col_offset, @@ -209,17 +213,19 @@ static int symtable_visit_alias(struct symtable *st, alias_ty); static int symtable_visit_comprehension(struct symtable *st, comprehension_ty); static int symtable_visit_keyword(struct symtable *st, keyword_ty); static int symtable_visit_params(struct symtable *st, asdl_arg_seq *args); +static int symtable_visit_annotation(struct symtable *st, expr_ty annotation); static int symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args); static int symtable_implicit_arg(struct symtable *st, int pos); -static int symtable_visit_annotations(struct symtable *st, arguments_ty, expr_ty); +static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty); static int symtable_visit_withitem(struct symtable *st, withitem_ty item); static int symtable_visit_match_case(struct symtable *st, match_case_ty m); static int symtable_visit_pattern(struct symtable *st, pattern_ty s); +static int symtable_raise_if_annotation_block(struct symtable *st, const char *, expr_ty); static identifier top = NULL, lambda = NULL, genexpr = NULL, listcomp = NULL, setcomp = NULL, dictcomp = NULL, - __class__ = NULL; + __class__ = NULL, _annotation = NULL; #define GET_IDENTIFIER(VAR) \ ((VAR) ? (VAR) : ((VAR) = PyUnicode_InternFromString(# VAR))) @@ -987,8 +993,17 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, /* The entry is owned by the stack. Borrow it for st_cur. */ Py_DECREF(ste); st->st_cur = ste; + + /* Annotation blocks shouldn't have any affect on the symbol table since in + * the compilation stage, they will all be transformed to strings. They are + * only created if future 'annotations' feature is activated. */ + if (block == AnnotationBlock) { + return 1; + } + if (block == ModuleBlock) st->st_global = st->st_cur->ste_symbols; + if (prev) { if (PyList_Append(prev->ste_children, (PyObject *)ste) < 0) { return 0; @@ -1190,7 +1205,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_SEQ(st, expr, s->v.FunctionDef.args->defaults); if (s->v.FunctionDef.args->kw_defaults) VISIT_SEQ_WITH_NULL(st, expr, s->v.FunctionDef.args->kw_defaults); - if (!symtable_visit_annotations(st, s->v.FunctionDef.args, + if (!symtable_visit_annotations(st, s, s->v.FunctionDef.args, s->v.FunctionDef.returns)) VISIT_QUIT(st, 0); if (s->v.FunctionDef.decorator_list) @@ -1273,7 +1288,10 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) else { VISIT(st, expr, s->v.AnnAssign.target); } - VISIT(st, expr, s->v.AnnAssign.annotation); + if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation)) { + VISIT_QUIT(st, 0); + } + if (s->v.AnnAssign.value) { VISIT(st, expr, s->v.AnnAssign.value); } @@ -1422,7 +1440,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) if (s->v.AsyncFunctionDef.args->kw_defaults) VISIT_SEQ_WITH_NULL(st, expr, s->v.AsyncFunctionDef.args->kw_defaults); - if (!symtable_visit_annotations(st, s->v.AsyncFunctionDef.args, + if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args, s->v.AsyncFunctionDef.returns)) VISIT_QUIT(st, 0); if (s->v.AsyncFunctionDef.decorator_list) @@ -1564,6 +1582,9 @@ symtable_visit_expr(struct symtable *st, expr_ty e) } switch (e->kind) { case NamedExpr_kind: + if (!symtable_raise_if_annotation_block(st, "named expression", e)) { + VISIT_QUIT(st, 0); + } if(!symtable_handle_namedexpr(st, e)) VISIT_QUIT(st, 0); break; @@ -1624,15 +1645,24 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT_QUIT(st, 0); break; case Yield_kind: + if (!symtable_raise_if_annotation_block(st, "yield expression", e)) { + VISIT_QUIT(st, 0); + } if (e->v.Yield.value) VISIT(st, expr, e->v.Yield.value); st->st_cur->ste_generator = 1; break; case YieldFrom_kind: + if (!symtable_raise_if_annotation_block(st, "yield expression", e)) { + VISIT_QUIT(st, 0); + } VISIT(st, expr, e->v.YieldFrom.value); st->st_cur->ste_generator = 1; break; case Await_kind: + if (!symtable_raise_if_annotation_block(st, "await expression", e)) { + VISIT_QUIT(st, 0); + } VISIT(st, expr, e->v.Await.value); st->st_cur->ste_coroutine = 1; break; @@ -1780,6 +1810,24 @@ symtable_visit_params(struct symtable *st, asdl_arg_seq *args) return 1; } +static int +symtable_visit_annotation(struct symtable *st, expr_ty annotation) +{ + int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS; + if (future_annotations && + !symtable_enter_block(st, GET_IDENTIFIER(_annotation), AnnotationBlock, + (void *)annotation, annotation->lineno, + annotation->col_offset, annotation->end_lineno, + annotation->end_col_offset)) { + VISIT_QUIT(st, 0); + } + VISIT(st, expr, annotation); + if (future_annotations && !symtable_exit_block(st)) { + VISIT_QUIT(st, 0); + } + return 1; +} + static int symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args) { @@ -1798,8 +1846,15 @@ symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args) } static int -symtable_visit_annotations(struct symtable *st, arguments_ty a, expr_ty returns) +symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns) { + int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS; + if (future_annotations && + !symtable_enter_block(st, GET_IDENTIFIER(_annotation), AnnotationBlock, + (void *)o, o->lineno, o->col_offset, o->end_lineno, + o->end_col_offset)) { + VISIT_QUIT(st, 0); + } if (a->posonlyargs && !symtable_visit_argannotations(st, a->posonlyargs)) return 0; if (a->args && !symtable_visit_argannotations(st, a->args)) @@ -1810,8 +1865,12 @@ symtable_visit_annotations(struct symtable *st, arguments_ty a, expr_ty returns) VISIT(st, expr, a->kwarg->annotation); if (a->kwonlyargs && !symtable_visit_argannotations(st, a->kwonlyargs)) return 0; - if (returns) - VISIT(st, expr, returns); + if (future_annotations && !symtable_exit_block(st)) { + VISIT_QUIT(st, 0); + } + if (returns && !symtable_visit_annotation(st, returns)) { + VISIT_QUIT(st, 0); + } return 1; } @@ -2033,6 +2092,21 @@ symtable_visit_dictcomp(struct symtable *st, expr_ty e) e->v.DictComp.value); } +static int +symtable_raise_if_annotation_block(struct symtable *st, const char *name, expr_ty e) +{ + if (st->st_cur->ste_type != AnnotationBlock) { + return 1; + } + + PyErr_Format(PyExc_SyntaxError, ANNOTATION_NOT_ALLOWED, name); + PyErr_RangedSyntaxLocationObject(st->st_filename, + e->lineno, + e->col_offset + 1, + e->end_lineno, + e->end_col_offset + 1); + return 0; +} struct symtable * _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, @@ -2051,7 +2125,14 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, _PyArena_Free(arena); return NULL; } - st = _PySymtable_Build(mod, filename, 0); + PyFutureFeatures *future = _PyFuture_FromAST(mod, filename); + if (future == NULL) { + _PyArena_Free(arena); + return NULL; + } + future->ff_features |= flags->cf_flags; + st = _PySymtable_Build(mod, filename, future); + PyObject_Free((void *)future); _PyArena_Free(arena); return st; }