diff --git a/Include/Python-ast.h b/Include/Python-ast.h index 8265bed8d6e..a5742ff0485 100644 --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -338,6 +338,7 @@ struct _expr { struct { constant value; + string kind; } Constant; struct { @@ -642,9 +643,9 @@ expr_ty _Py_FormattedValue(expr_ty value, int conversion, expr_ty format_spec, #define JoinedStr(a0, a1, a2, a3, a4, a5) _Py_JoinedStr(a0, a1, a2, a3, a4, a5) expr_ty _Py_JoinedStr(asdl_seq * values, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); -#define Constant(a0, a1, a2, a3, a4, a5) _Py_Constant(a0, a1, a2, a3, a4, a5) -expr_ty _Py_Constant(constant value, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); +#define Constant(a0, a1, a2, a3, a4, a5, a6) _Py_Constant(a0, a1, a2, a3, a4, a5, a6) +expr_ty _Py_Constant(constant value, string kind, int lineno, int col_offset, + int end_lineno, int end_col_offset, PyArena *arena); #define Attribute(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Attribute(a0, a1, a2, a3, a4, a5, a6, a7) expr_ty _Py_Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int col_offset, int end_lineno, int diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 609c8b2c6a7..a7ee0da38d4 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -326,7 +326,7 @@ def test_field_attr_writable(self): def test_classattrs(self): x = ast.Num() - self.assertEqual(x._fields, ('value',)) + self.assertEqual(x._fields, ('value', 'kind')) with self.assertRaises(AttributeError): x.value @@ -349,12 +349,12 @@ def test_classattrs(self): x = ast.Num(42, lineno=0) self.assertEqual(x.lineno, 0) - self.assertEqual(x._fields, ('value',)) + self.assertEqual(x._fields, ('value', 'kind')) self.assertEqual(x.value, 42) self.assertEqual(x.n, 42) - self.assertRaises(TypeError, ast.Num, 1, 2) - self.assertRaises(TypeError, ast.Num, 1, 2, lineno=0) + self.assertRaises(TypeError, ast.Num, 1, None, 2) + self.assertRaises(TypeError, ast.Num, 1, None, 2, lineno=0) self.assertEqual(ast.Num(42).n, 42) self.assertEqual(ast.Num(4.25).n, 4.25) @@ -556,6 +556,7 @@ def bad_normalize(*args): class ASTHelpers_Test(unittest.TestCase): + maxDiff = None def test_parse(self): a = ast.parse('foo(1 + 1)') @@ -574,18 +575,18 @@ def test_dump(self): node = ast.parse('spam(eggs, "and cheese")') self.assertEqual(ast.dump(node), "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), " - "args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')], " + "args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese', kind=None)], " "keywords=[]))], type_ignores=[])" ) self.assertEqual(ast.dump(node, annotate_fields=False), "Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), " - "Constant('and cheese')], []))], [])" + "Constant('and cheese', None)], []))], [])" ) self.assertEqual(ast.dump(node, include_attributes=True), "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=4), " "args=[Name(id='eggs', ctx=Load(), lineno=1, col_offset=5, " - "end_lineno=1, end_col_offset=9), Constant(value='and cheese', " + "end_lineno=1, end_col_offset=9), Constant(value='and cheese', kind=None, " "lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], keywords=[], " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24), " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)], type_ignores=[])" @@ -595,7 +596,7 @@ def test_copy_location(self): src = ast.parse('1 + 1', mode='eval') src.body.right = ast.copy_location(ast.Num(2), src.body.right) self.assertEqual(ast.dump(src, include_attributes=True), - 'Expression(body=BinOp(left=Constant(value=1, lineno=1, col_offset=0, ' + 'Expression(body=BinOp(left=Constant(value=1, kind=None, lineno=1, col_offset=0, ' 'end_lineno=1, end_col_offset=1), op=Add(), right=Constant(value=2, ' 'lineno=1, col_offset=4, end_lineno=1, end_col_offset=5), lineno=1, ' 'col_offset=0, end_lineno=1, end_col_offset=5))' @@ -610,7 +611,7 @@ def test_fix_missing_locations(self): self.assertEqual(ast.dump(src, include_attributes=True), "Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=5), " - "args=[Constant(value='spam', lineno=1, col_offset=6, end_lineno=1, " + "args=[Constant(value='spam', kind=None, lineno=1, col_offset=6, end_lineno=1, " "end_col_offset=12)], keywords=[], lineno=1, col_offset=0, end_lineno=1, " "end_col_offset=13), lineno=1, col_offset=0, end_lineno=1, " "end_col_offset=13), Expr(value=Call(func=Name(id='spam', ctx=Load(), " @@ -625,8 +626,8 @@ def test_increment_lineno(self): src = ast.parse('1 + 1', mode='eval') self.assertEqual(ast.increment_lineno(src, n=3), src) self.assertEqual(ast.dump(src, include_attributes=True), - 'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, ' - 'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, ' + 'Expression(body=BinOp(left=Constant(value=1, kind=None, lineno=4, col_offset=0, ' + 'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, kind=None, ' 'lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, ' 'col_offset=0, end_lineno=4, end_col_offset=5))' ) @@ -634,8 +635,8 @@ def test_increment_lineno(self): src = ast.parse('1 + 1', mode='eval') self.assertEqual(ast.increment_lineno(src.body, n=3), src.body) self.assertEqual(ast.dump(src, include_attributes=True), - 'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, ' - 'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, ' + 'Expression(body=BinOp(left=Constant(value=1, kind=None, lineno=4, col_offset=0, ' + 'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, kind=None, ' 'lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, ' 'col_offset=0, end_lineno=4, end_col_offset=5))' ) @@ -654,7 +655,7 @@ def test_iter_child_nodes(self): self.assertEqual(next(iterator).value, 23) self.assertEqual(next(iterator).value, 42) self.assertEqual(ast.dump(next(iterator)), - "keyword(arg='eggs', value=Constant(value='leek'))" + "keyword(arg='eggs', value=Constant(value='leek', kind=None))" ) def test_get_docstring(self): @@ -1283,6 +1284,23 @@ def test_literal_eval(self): self.assertEqual(ast.literal_eval(binop), 10+20j) + def test_string_kind(self): + c = ast.parse('"x"', mode='eval').body + self.assertEqual(c.value, "x") + self.assertEqual(c.kind, None) + + c = ast.parse('u"x"', mode='eval').body + self.assertEqual(c.value, "x") + self.assertEqual(c.kind, "u") + + c = ast.parse('r"x"', mode='eval').body + self.assertEqual(c.value, "x") + self.assertEqual(c.kind, None) + + c = ast.parse('b"x"', mode='eval').body + self.assertEqual(c.value, b"x") + self.assertEqual(c.kind, None) + class EndPositionTests(unittest.TestCase): """Tests for end position of AST nodes. @@ -1606,38 +1624,38 @@ def main(): #### EVERYTHING BELOW IS GENERATED BY python Lib/test/test_ast.py -g ##### exec_results = [ -('Module', [('Expr', (1, 0), ('Constant', (1, 0), None))], []), -('Module', [('Expr', (1, 0), ('Constant', (1, 0), 'module docstring'))], []), +('Module', [('Expr', (1, 0), ('Constant', (1, 0), None, None))], []), +('Module', [('Expr', (1, 0), ('Constant', (1, 0), 'module docstring', None))], []), ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (1, 9))], [], None, None)], []), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (1, 9), ('Constant', (1, 9), 'function docstring'))], [], None, None)], []), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (1, 9), ('Constant', (1, 9), 'function docstring', None))], [], None, None)], []), ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None, None)], None, [], [], None, []), [('Pass', (1, 10))], [], None, None)], []), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None, None)], None, [], [], None, [('Constant', (1, 8), 0)]), [('Pass', (1, 12))], [], None, None)], []), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None, None)], None, [], [], None, [('Constant', (1, 8), 0, None)]), [('Pass', (1, 12))], [], None, None)], []), ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], ('arg', (1, 7), 'args', None, None), [], [], None, []), [('Pass', (1, 14))], [], None, None)], []), ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], ('arg', (1, 8), 'kwargs', None, None), []), [('Pass', (1, 17))], [], None, None)], []), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None, None), ('arg', (1, 9), 'b', None, None), ('arg', (1, 14), 'c', None, None), ('arg', (1, 22), 'd', None, None), ('arg', (1, 28), 'e', None, None)], ('arg', (1, 35), 'args', None, None), [('arg', (1, 41), 'f', None, None)], [('Constant', (1, 43), 42)], ('arg', (1, 49), 'kwargs', None, None), [('Constant', (1, 11), 1), ('Constant', (1, 16), None), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])]), [('Expr', (1, 58), ('Constant', (1, 58), 'doc for f()'))], [], None, None)], []), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None, None), ('arg', (1, 9), 'b', None, None), ('arg', (1, 14), 'c', None, None), ('arg', (1, 22), 'd', None, None), ('arg', (1, 28), 'e', None, None)], ('arg', (1, 35), 'args', None, None), [('arg', (1, 41), 'f', None, None)], [('Constant', (1, 43), 42, None)], ('arg', (1, 49), 'kwargs', None, None), [('Constant', (1, 11), 1, None), ('Constant', (1, 16), None, None), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])]), [('Expr', (1, 58), ('Constant', (1, 58), 'doc for f()', None))], [], None, None)], []), ('Module', [('ClassDef', (1, 0), 'C', [], [], [('Pass', (1, 8))], [])], []), -('Module', [('ClassDef', (1, 0), 'C', [], [], [('Expr', (1, 9), ('Constant', (1, 9), 'docstring for class C'))], [])], []), +('Module', [('ClassDef', (1, 0), 'C', [], [], [('Expr', (1, 9), ('Constant', (1, 9), 'docstring for class C', None))], [])], []), ('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [], [('Pass', (1, 17))], [])], []), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Return', (1, 8), ('Constant', (1, 15), 1))], [], None, None)], []), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Return', (1, 8), ('Constant', (1, 15), 1, None))], [], None, None)], []), ('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])], []), -('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Constant', (1, 4), 1), None)], []), +('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Constant', (1, 4), 1, None), None)], []), ('Module', [('Assign', (1, 0), [('Tuple', (1, 0), [('Name', (1, 0), 'a', ('Store',)), ('Name', (1, 2), 'b', ('Store',))], ('Store',))], ('Name', (1, 6), 'c', ('Load',)), None)], []), ('Module', [('Assign', (1, 0), [('Tuple', (1, 0), [('Name', (1, 1), 'a', ('Store',)), ('Name', (1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 8), 'c', ('Load',)), None)], []), ('Module', [('Assign', (1, 0), [('List', (1, 0), [('Name', (1, 1), 'a', ('Store',)), ('Name', (1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 8), 'c', ('Load',)), None)], []), -('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Constant', (1, 5), 1))], []), +('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Constant', (1, 5), 1, None))], []), ('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [], None)], []), ('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])], []), ('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])], []), ('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',)))], [('Pass', (1, 13))], None)], []), ('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',))), ('withitem', ('Name', (1, 13), 'z', ('Load',)), ('Name', (1, 18), 'q', ('Store',)))], [('Pass', (1, 21))], None)], []), -('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Constant', (1, 16), 'string')], []), None)], []), +('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Constant', (1, 16), 'string', None)], []), None)], []), ('Module', [('Try', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [], [])], []), ('Module', [('Try', (1, 0), [('Pass', (2, 2))], [], [], [('Pass', (4, 2))])], []), ('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)], []), ('Module', [('Import', (1, 0), [('alias', 'sys', None)])], []), ('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)], []), ('Module', [('Global', (1, 0), ['v'])], []), -('Module', [('Expr', (1, 0), ('Constant', (1, 0), 1))], []), +('Module', [('Expr', (1, 0), ('Constant', (1, 0), 1, None))], []), ('Module', [('Pass', (1, 0))], []), ('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Break', (1, 11))], [], None)], []), ('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Continue', (1, 11))], [], None)], []), @@ -1649,11 +1667,11 @@ def main(): ('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'v', ('Store',)), ('Name', (1, 13), 'w', ('Store',))], ('Store',)), ('Name', (1, 18), 'x', ('Load',)), [], 0)]))], []), ('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 12), 'x', ('Load',)), [('Name', (1, 17), 'g', ('Load',))], 0)]))], []), ('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7), [('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 9), 'm', ('Store',))], ('Store',)), ('Name', (1, 14), 'x', ('Load',)), [], 0)]))], []), -('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('Constant', (2, 1), 'async function')), ('Expr', (3, 1), ('Await', (3, 1), ('Call', (3, 7), ('Name', (3, 7), 'something', ('Load',)), [], [])))], [], None, None)], []), -('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('AsyncFor', (2, 1), ('Name', (2, 11), 'e', ('Store',)), ('Name', (2, 16), 'i', ('Load',)), [('Expr', (2, 19), ('Constant', (2, 19), 1))], [('Expr', (3, 7), ('Constant', (3, 7), 2))], None)], [], None, None)], []), -('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('AsyncWith', (2, 1), [('withitem', ('Name', (2, 12), 'a', ('Load',)), ('Name', (2, 17), 'b', ('Store',)))], [('Expr', (2, 20), ('Constant', (2, 20), 1))], None)], [], None, None)], []), -('Module', [('Expr', (1, 0), ('Dict', (1, 0), [None, ('Constant', (1, 10), 2)], [('Dict', (1, 3), [('Constant', (1, 4), 1)], [('Constant', (1, 6), 2)]), ('Constant', (1, 12), 3)]))], []), -('Module', [('Expr', (1, 0), ('Set', (1, 0), [('Starred', (1, 1), ('Set', (1, 2), [('Constant', (1, 3), 1), ('Constant', (1, 6), 2)]), ('Load',)), ('Constant', (1, 10), 3)]))], []), +('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('Constant', (2, 1), 'async function', None)), ('Expr', (3, 1), ('Await', (3, 1), ('Call', (3, 7), ('Name', (3, 7), 'something', ('Load',)), [], [])))], [], None, None)], []), +('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('AsyncFor', (2, 1), ('Name', (2, 11), 'e', ('Store',)), ('Name', (2, 16), 'i', ('Load',)), [('Expr', (2, 19), ('Constant', (2, 19), 1, None))], [('Expr', (3, 7), ('Constant', (3, 7), 2, None))], None)], [], None, None)], []), +('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('AsyncWith', (2, 1), [('withitem', ('Name', (2, 12), 'a', ('Load',)), ('Name', (2, 17), 'b', ('Store',)))], [('Expr', (2, 20), ('Constant', (2, 20), 1, None))], None)], [], None, None)], []), +('Module', [('Expr', (1, 0), ('Dict', (1, 0), [None, ('Constant', (1, 10), 2, None)], [('Dict', (1, 3), [('Constant', (1, 4), 1, None)], [('Constant', (1, 6), 2, None)]), ('Constant', (1, 12), 3, None)]))], []), +('Module', [('Expr', (1, 0), ('Set', (1, 0), [('Starred', (1, 1), ('Set', (1, 2), [('Constant', (1, 3), 1, None), ('Constant', (1, 6), 2, None)]), ('Load',)), ('Constant', (1, 10), 3, None)]))], []), ('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('ListComp', (2, 1), ('Name', (2, 2), 'i', ('Load',)), [('comprehension', ('Name', (2, 14), 'b', ('Store',)), ('Name', (2, 19), 'c', ('Load',)), [], 1)]))], [], None, None)], []), ('Module', [('FunctionDef', (3, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (3, 9))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])], None, None)], []), ('Module', [('AsyncFunctionDef', (3, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (3, 15))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])], None, None)], []), @@ -1661,18 +1679,18 @@ def main(): ('Module', [('FunctionDef', (2, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (2, 9))], [('Call', (1, 1), ('Name', (1, 1), 'deco', ('Load',)), [('GeneratorExp', (1, 5), ('Name', (1, 6), 'a', ('Load',)), [('comprehension', ('Name', (1, 12), 'a', ('Store',)), ('Name', (1, 17), 'b', ('Load',)), [], 0)])], [])], None, None)], []), ] single_results = [ -('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Constant', (1, 0), 1), ('Add',), ('Constant', (1, 2), 2)))]), +('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Constant', (1, 0), 1, None), ('Add',), ('Constant', (1, 2), 2, None)))]), ] eval_results = [ -('Expression', ('Constant', (1, 0), None)), +('Expression', ('Constant', (1, 0), None, None)), ('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])), ('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))), ('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))), -('Expression', ('Lambda', (1, 0), ('arguments', [], None, [], [], None, []), ('Constant', (1, 7), None))), -('Expression', ('Dict', (1, 0), [('Constant', (1, 2), 1)], [('Constant', (1, 4), 2)])), +('Expression', ('Lambda', (1, 0), ('arguments', [], None, [], [], None, []), ('Constant', (1, 7), None, None))), +('Expression', ('Dict', (1, 0), [('Constant', (1, 2), 1, None)], [('Constant', (1, 4), 2, None)])), ('Expression', ('Dict', (1, 0), [], [])), -('Expression', ('Set', (1, 0), [('Constant', (1, 1), None)])), -('Expression', ('Dict', (1, 0), [('Constant', (2, 6), 1)], [('Constant', (4, 10), 2)])), +('Expression', ('Set', (1, 0), [('Constant', (1, 1), None, None)])), +('Expression', ('Dict', (1, 0), [('Constant', (2, 6), 1, None)], [('Constant', (4, 10), 2, None)])), ('Expression', ('ListComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))], 0)])), ('Expression', ('GeneratorExp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))], 0)])), ('Expression', ('ListComp', (1, 0), ('Tuple', (1, 1), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [], 0)])), @@ -1684,19 +1702,19 @@ def main(): ('Expression', ('GeneratorExp', (1, 0), ('Tuple', (1, 1), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [], 0)])), ('Expression', ('GeneratorExp', (1, 0), ('Tuple', (1, 1), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 12), 'a', ('Store',)), ('Name', (1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 20), 'c', ('Load',)), [], 0)])), ('Expression', ('GeneratorExp', (1, 0), ('Tuple', (1, 1), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11), [('Name', (1, 12), 'a', ('Store',)), ('Name', (1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 20), 'c', ('Load',)), [], 0)])), -('Expression', ('Compare', (1, 0), ('Constant', (1, 0), 1), [('Lt',), ('Lt',)], [('Constant', (1, 4), 2), ('Constant', (1, 8), 3)])), -('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('Constant', (1, 2), 1), ('Constant', (1, 4), 2), ('Starred', (1, 10), ('Name', (1, 11), 'd', ('Load',)), ('Load',))], [('keyword', 'c', ('Constant', (1, 8), 3)), ('keyword', None, ('Name', (1, 15), 'e', ('Load',)))])), +('Expression', ('Compare', (1, 0), ('Constant', (1, 0), 1, None), [('Lt',), ('Lt',)], [('Constant', (1, 4), 2, None), ('Constant', (1, 8), 3, None)])), +('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('Constant', (1, 2), 1, None), ('Constant', (1, 4), 2, None), ('Starred', (1, 10), ('Name', (1, 11), 'd', ('Load',)), ('Load',))], [('keyword', 'c', ('Constant', (1, 8), 3, None)), ('keyword', None, ('Name', (1, 15), 'e', ('Load',)))])), ('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('GeneratorExp', (1, 1), ('Name', (1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 8), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Load',)), [], 0)])], [])), -('Expression', ('Constant', (1, 0), 10)), -('Expression', ('Constant', (1, 0), 'string')), +('Expression', ('Constant', (1, 0), 10, None)), +('Expression', ('Constant', (1, 0), 'string', None)), ('Expression', ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',))), ('Expression', ('Subscript', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))), ('Expression', ('Name', (1, 0), 'v', ('Load',))), -('Expression', ('List', (1, 0), [('Constant', (1, 1), 1), ('Constant', (1, 3), 2), ('Constant', (1, 5), 3)], ('Load',))), +('Expression', ('List', (1, 0), [('Constant', (1, 1), 1, None), ('Constant', (1, 3), 2, None), ('Constant', (1, 5), 3, None)], ('Load',))), ('Expression', ('List', (1, 0), [], ('Load',))), -('Expression', ('Tuple', (1, 0), [('Constant', (1, 0), 1), ('Constant', (1, 2), 2), ('Constant', (1, 4), 3)], ('Load',))), -('Expression', ('Tuple', (1, 0), [('Constant', (1, 1), 1), ('Constant', (1, 3), 2), ('Constant', (1, 5), 3)], ('Load',))), +('Expression', ('Tuple', (1, 0), [('Constant', (1, 0), 1, None), ('Constant', (1, 2), 2, None), ('Constant', (1, 4), 3, None)], ('Load',))), +('Expression', ('Tuple', (1, 0), [('Constant', (1, 1), 1, None), ('Constant', (1, 3), 2, None), ('Constant', (1, 5), 3, None)], ('Load',))), ('Expression', ('Tuple', (1, 0), [], ('Load',))), -('Expression', ('Call', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8), ('Attribute', (1, 8), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Constant', (1, 12), 1), ('Constant', (1, 14), 2), None), ('Load',))], [])), +('Expression', ('Call', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8), ('Attribute', (1, 8), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Constant', (1, 12), 1, None), ('Constant', (1, 14), 2, None), None), ('Load',))], [])), ] main() diff --git a/Misc/NEWS.d/next/Library/2019-03-12-21-02-55.bpo-36280.mOd3iH.rst b/Misc/NEWS.d/next/Library/2019-03-12-21-02-55.bpo-36280.mOd3iH.rst new file mode 100644 index 00000000000..e97285431e5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-03-12-21-02-55.bpo-36280.mOd3iH.rst @@ -0,0 +1,2 @@ +Add a kind field to ast.Constant. It is 'u' if the literal has a 'u' prefix +(i.e. a Python 2 style unicode literal), else None. diff --git a/Parser/Python.asdl b/Parser/Python.asdl index 1ccd2ca223f..c32b65a52fe 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -78,7 +78,7 @@ module Python | Call(expr func, expr* args, keyword* keywords) | FormattedValue(expr value, int? conversion, expr? format_spec) | JoinedStr(expr* values) - | Constant(constant value) + | Constant(constant value, string? kind) -- the following expression can appear in assignment context | Attribute(expr value, identifier attr, expr_context ctx) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 0411f9f07f2..d0416eb639c 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -324,8 +324,10 @@ static char *JoinedStr_fields[]={ "values", }; static PyTypeObject *Constant_type; +_Py_IDENTIFIER(kind); static char *Constant_fields[]={ "value", + "kind", }; static PyTypeObject *Attribute_type; _Py_IDENTIFIER(attr); @@ -950,7 +952,7 @@ static int init_types(void) if (!FormattedValue_type) return 0; JoinedStr_type = make_type("JoinedStr", expr_type, JoinedStr_fields, 1); if (!JoinedStr_type) return 0; - Constant_type = make_type("Constant", expr_type, Constant_fields, 1); + Constant_type = make_type("Constant", expr_type, Constant_fields, 2); if (!Constant_type) return 0; Attribute_type = make_type("Attribute", expr_type, Attribute_fields, 3); if (!Attribute_type) return 0; @@ -2287,8 +2289,8 @@ JoinedStr(asdl_seq * values, int lineno, int col_offset, int end_lineno, int } expr_ty -Constant(constant value, int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena) +Constant(constant value, string kind, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) { expr_ty p; if (!value) { @@ -2301,6 +2303,7 @@ Constant(constant value, int lineno, int col_offset, int end_lineno, int return NULL; p->kind = Constant_kind; p->v.Constant.value = value; + p->v.Constant.kind = kind; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -3507,6 +3510,11 @@ ast2obj_expr(void* _o) if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_string(o->v.Constant.kind); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_kind, value) == -1) + goto failed; + Py_DECREF(value); break; case Attribute_kind: result = PyType_GenericNew(Attribute_type, NULL, NULL); @@ -7224,6 +7232,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } if (isinstance) { constant value; + string kind; if (_PyObject_LookupAttrId(obj, &PyId_value, &tmp) < 0) { return 1; @@ -7238,8 +7247,21 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = Constant(value, lineno, col_offset, end_lineno, end_col_offset, - arena); + if (_PyObject_LookupAttrId(obj, &PyId_kind, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + kind = NULL; + } + else { + int res; + res = obj2ast_string(tmp, &kind, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = Constant(value, kind, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } diff --git a/Python/ast.c b/Python/ast.c index d4ee1b351fe..971b8ddc8c2 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -2313,13 +2313,13 @@ ast_for_atom(struct compiling *c, const node *n) size_t len = strlen(s); if (len >= 4 && len <= 5) { if (!strcmp(s, "None")) - return Constant(Py_None, LINENO(n), n->n_col_offset, + return Constant(Py_None, NULL, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); if (!strcmp(s, "True")) - return Constant(Py_True, LINENO(n), n->n_col_offset, + return Constant(Py_True, NULL, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); if (!strcmp(s, "False")) - return Constant(Py_False, LINENO(n), n->n_col_offset, + return Constant(Py_False, NULL, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } name = new_identifier(s, c); @@ -2374,11 +2374,11 @@ ast_for_atom(struct compiling *c, const node *n) Py_DECREF(pynum); return NULL; } - return Constant(pynum, LINENO(n), n->n_col_offset, + return Constant(pynum, NULL, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } case ELLIPSIS: /* Ellipsis */ - return Constant(Py_Ellipsis, LINENO(n), n->n_col_offset, + return Constant(Py_Ellipsis, NULL, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); case LPAR: /* some parenthesized expressions */ ch = CHILD(n, 1); @@ -5388,18 +5388,57 @@ FstringParser_Dealloc(FstringParser *state) ExprList_Dealloc(&state->expr_list); } +/* Constants for the following */ +static PyObject *u_kind; + +/* Compute 'kind' field for string Constant (either 'u' or None) */ +static PyObject * +make_kind(struct compiling *c, const node *n) +{ + char *s = NULL; + PyObject *kind = NULL; + + /* Find the first string literal, if any */ + while (TYPE(n) != STRING) { + if (NCH(n) == 0) + return NULL; + n = CHILD(n, 0); + } + REQ(n, STRING); + + /* If it starts with 'u', return a PyUnicode "u" string */ + s = STR(n); + if (s && *s == 'u') { + if (!u_kind) { + u_kind = PyUnicode_InternFromString("u"); + if (!u_kind) + return NULL; + } + kind = u_kind; + if (PyArena_AddPyObject(c->c_arena, kind) < 0) { + return NULL; + } + Py_INCREF(kind); + } + return kind; +} + /* Make a Constant node, but decref the PyUnicode object being added. */ static expr_ty make_str_node_and_del(PyObject **str, struct compiling *c, const node* n) { PyObject *s = *str; + PyObject *kind = NULL; *str = NULL; assert(PyUnicode_CheckExact(s)); if (PyArena_AddPyObject(c->c_arena, s) < 0) { Py_DECREF(s); return NULL; } - return Constant(s, LINENO(n), n->n_col_offset, + kind = make_kind(c, n); + if (kind == NULL && PyErr_Occurred()) + return NULL; + return Constant(s, kind, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } @@ -5774,7 +5813,7 @@ parsestrplus(struct compiling *c, const node *n) /* Just return the bytes object and we're done. */ if (PyArena_AddPyObject(c->c_arena, bytes_str) < 0) goto error; - return Constant(bytes_str, LINENO(n), n->n_col_offset, + return Constant(bytes_str, NULL, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); }