bpo-33416: Add end positions to Python AST (GH-11605)

The majority of this PR is tediously passing `end_lineno` and `end_col_offset` everywhere. Here are non-trivial points:
* It is not possible to reconstruct end positions in AST "on the fly", some information is lost after an AST node is constructed, so we need two more attributes for every AST node `end_lineno` and `end_col_offset`.
* I add end position information to both CST and AST.  Although it may be technically possible to avoid adding end positions to CST, the code becomes more cumbersome and less efficient.
* Since the end position is not known for non-leaf CST nodes while the next token is added, this requires a bit of extra care (see `_PyNode_FinalizeEndPos`). Unless I made some mistake, the algorithm should be linear.
* For statements, I "trim" the end position of suites to not include the terminal newlines and dedent (this seems to be what people would expect), for example in
  ```python
  class C:
      pass

  pass
  ```
  the end line and end column for the class definition is (2, 8).
* For `end_col_offset` I use the common Python convention for indexing, for example for `pass` the `end_col_offset` is 4 (not 3), so that `[0:4]` gives one the source code that corresponds to the node.
* I added a helper function `ast.get_source_segment()`, to get source text segment corresponding to a given AST node. It is also useful for testing.

An (inevitable) downside of this PR is that AST now takes almost 25% more memory. I think however it is probably justified by the benefits.

Doc/library/ast.rst

@@ -61,13 +61,21 @@ Node classes
 
    .. attribute:: lineno
                   col_offset
+                  end_lineno
+                  end_col_offset
 
       Instances of :class:`ast.expr` and :class:`ast.stmt` subclasses have
-      :attr:`lineno` and :attr:`col_offset` attributes.  The :attr:`lineno` is
-      the line number of source text (1-indexed so the first line is line 1) and
-      the :attr:`col_offset` is the UTF-8 byte offset of the first token that
-      generated the node.  The UTF-8 offset is recorded because the parser uses
-      UTF-8 internally.
+      :attr:`lineno`, :attr:`col_offset`, :attr:`lineno`, and :attr:`col_offset`
+      attributes.  The :attr:`lineno` and :attr:`end_lineno` are the first and
+      last line numbers of source text span (1-indexed so the first line is line 1)
+      and the :attr:`col_offset` and :attr:`end_col_offset` are the corresponding
+      UTF-8 byte offsets of the first and last tokens that generated the node.
+      The UTF-8 offset is recorded because the parser uses UTF-8 internally.
+
+      Note that the end positions are not required by the compiler and are
+      therefore optional. The end offset is *after* the last symbol, for example
+      one can get the source segment of a one-line expression node using
+      ``source_line[node.col_offset : node.end_col_offset]``.
 
    The constructor of a class :class:`ast.T` parses its arguments as follows:
 
@@ -162,6 +170,18 @@ and classes for traversing abstract syntax trees:
       :class:`AsyncFunctionDef` is now supported.
 
 
+.. function:: get_source_segment(source, node, *, padded=False)
+
+   Get source code segment of the *source* that generated *node*.
+   If some location information (:attr:`lineno`, :attr:`end_lineno`,
+   :attr:`col_offset`, or :attr:`end_col_offset`) is missing, return ``None``.
+
+   If *padded* is ``True``, the first line of a multi-line statement will
+   be padded with spaces to match its original position.
+
+   .. versionadded:: 3.8
+
+
 .. function:: fix_missing_locations(node)
 
    When you compile a node tree with :func:`compile`, the compiler expects
@@ -173,14 +193,16 @@ and classes for traversing abstract syntax trees:
 
 .. function:: increment_lineno(node, n=1)
 
-   Increment the line number of each node in the tree starting at *node* by *n*.
-   This is useful to "move code" to a different location in a file.
+   Increment the line number and end line number of each node in the tree
+   starting at *node* by *n*. This is useful to "move code" to a different
+   location in a file.
 
 
 .. function:: copy_location(new_node, old_node)
 
-   Copy source location (:attr:`lineno` and :attr:`col_offset`) from *old_node*
-   to *new_node* if possible, and return *new_node*.
+   Copy source location (:attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`,
+   and :attr:`end_col_offset`) from *old_node* to *new_node* if possible,
+   and return *new_node*.
 
 
 .. function:: iter_fields(node)

Include/Python-ast.h

@@ -210,6 +210,8 @@ struct _stmt {
     } v;
     int lineno;
     int col_offset;
+    int end_lineno;
+    int end_col_offset;
 };
 
 enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
@@ -353,6 +355,8 @@ struct _expr {
     } v;
     int lineno;
     int col_offset;
+    int end_lineno;
+    int end_col_offset;
 };
 
 enum _slice_kind {Slice_kind=1, ExtSlice_kind=2, Index_kind=3};
@@ -396,6 +400,8 @@ struct _excepthandler {
     } v;
     int lineno;
     int col_offset;
+    int end_lineno;
+    int end_col_offset;
 };
 
 struct _arguments {
@@ -412,6 +418,8 @@ struct _arg {
     expr_ty annotation;
     int lineno;
     int col_offset;
+    int end_lineno;
+    int end_col_offset;
 };
 
 struct _keyword {
@@ -430,6 +438,7 @@ struct _withitem {
 };
 
 
+// Note: these macros affect function definitions, not only call sites.
 #define Module(a0, a1) _Py_Module(a0, a1)
 mod_ty _Py_Module(asdl_seq * body, PyArena *arena);
 #define Interactive(a0, a1) _Py_Interactive(a0, a1)
@@ -438,152 +447,188 @@ mod_ty _Py_Interactive(asdl_seq * body, PyArena *arena);
 mod_ty _Py_Expression(expr_ty body, PyArena *arena);
 #define Suite(a0, a1) _Py_Suite(a0, a1)
 mod_ty _Py_Suite(asdl_seq * body, PyArena *arena);
-#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7)
+#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9)
 stmt_ty _Py_FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
                         asdl_seq * decorator_list, expr_ty returns, int lineno,
-                        int col_offset, PyArena *arena);
-#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7)
+                        int col_offset, int end_lineno, int end_col_offset,
+                        PyArena *arena);
+#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9)
 stmt_ty _Py_AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq *
                              body, asdl_seq * decorator_list, expr_ty returns,
-                             int lineno, int col_offset, PyArena *arena);
-#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7)
+                             int lineno, int col_offset, int end_lineno, int
+                             end_col_offset, PyArena *arena);
+#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9)
 stmt_ty _Py_ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords,
                      asdl_seq * body, asdl_seq * decorator_list, int lineno,
-                     int col_offset, PyArena *arena);
-#define Return(a0, a1, a2, a3) _Py_Return(a0, a1, a2, a3)
-stmt_ty _Py_Return(expr_ty value, int lineno, int col_offset, PyArena *arena);
-#define Delete(a0, a1, a2, a3) _Py_Delete(a0, a1, a2, a3)
-stmt_ty _Py_Delete(asdl_seq * targets, int lineno, int col_offset, PyArena
-                   *arena);
-#define Assign(a0, a1, a2, a3, a4) _Py_Assign(a0, a1, a2, a3, a4)
+                     int col_offset, int end_lineno, int end_col_offset,
+                     PyArena *arena);
+#define Return(a0, a1, a2, a3, a4, a5) _Py_Return(a0, a1, a2, a3, a4, a5)
+stmt_ty _Py_Return(expr_ty value, int lineno, int col_offset, int end_lineno,
+                   int end_col_offset, PyArena *arena);
+#define Delete(a0, a1, a2, a3, a4, a5) _Py_Delete(a0, a1, a2, a3, a4, a5)
+stmt_ty _Py_Delete(asdl_seq * targets, int lineno, int col_offset, int
+                   end_lineno, int end_col_offset, PyArena *arena);
+#define Assign(a0, a1, a2, a3, a4, a5, a6) _Py_Assign(a0, a1, a2, a3, a4, a5, a6)
 stmt_ty _Py_Assign(asdl_seq * targets, expr_ty value, int lineno, int
-                   col_offset, PyArena *arena);
-#define AugAssign(a0, a1, a2, a3, a4, a5) _Py_AugAssign(a0, a1, a2, a3, a4, a5)
+                   col_offset, int end_lineno, int end_col_offset, PyArena
+                   *arena);
+#define AugAssign(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AugAssign(a0, a1, a2, a3, a4, a5, a6, a7)
 stmt_ty _Py_AugAssign(expr_ty target, operator_ty op, expr_ty value, int
-                      lineno, int col_offset, PyArena *arena);
-#define AnnAssign(a0, a1, a2, a3, a4, a5, a6) _Py_AnnAssign(a0, a1, a2, a3, a4, a5, a6)
+                      lineno, int col_offset, int end_lineno, int
+                      end_col_offset, PyArena *arena);
+#define AnnAssign(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_AnnAssign(a0, a1, a2, a3, a4, a5, a6, a7, a8)
 stmt_ty _Py_AnnAssign(expr_ty target, expr_ty annotation, expr_ty value, int
-                      simple, int lineno, int col_offset, PyArena *arena);
-#define For(a0, a1, a2, a3, a4, a5, a6) _Py_For(a0, a1, a2, a3, a4, a5, a6)
+                      simple, int lineno, int col_offset, int end_lineno, int
+                      end_col_offset, PyArena *arena);
+#define For(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_For(a0, a1, a2, a3, a4, a5, a6, a7, a8)
 stmt_ty _Py_For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq *
-                orelse, int lineno, int col_offset, PyArena *arena);
-#define AsyncFor(a0, a1, a2, a3, a4, a5, a6) _Py_AsyncFor(a0, a1, a2, a3, a4, a5, a6)
+                orelse, int lineno, int col_offset, int end_lineno, int
+                end_col_offset, PyArena *arena);
+#define AsyncFor(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_AsyncFor(a0, a1, a2, a3, a4, a5, a6, a7, a8)
 stmt_ty _Py_AsyncFor(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq *
-                     orelse, int lineno, int col_offset, PyArena *arena);
-#define While(a0, a1, a2, a3, a4, a5) _Py_While(a0, a1, a2, a3, a4, a5)
+                     orelse, int lineno, int col_offset, int end_lineno, int
+                     end_col_offset, PyArena *arena);
+#define While(a0, a1, a2, a3, a4, a5, a6, a7) _Py_While(a0, a1, a2, a3, a4, a5, a6, a7)
 stmt_ty _Py_While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
-                  int col_offset, PyArena *arena);
-#define If(a0, a1, a2, a3, a4, a5) _Py_If(a0, a1, a2, a3, a4, a5)
+                  int col_offset, int end_lineno, int end_col_offset, PyArena
+                  *arena);
+#define If(a0, a1, a2, a3, a4, a5, a6, a7) _Py_If(a0, a1, a2, a3, a4, a5, a6, a7)
 stmt_ty _Py_If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
-               int col_offset, PyArena *arena);
-#define With(a0, a1, a2, a3, a4) _Py_With(a0, a1, a2, a3, a4)
+               int col_offset, int end_lineno, int end_col_offset, PyArena
+               *arena);
+#define With(a0, a1, a2, a3, a4, a5, a6) _Py_With(a0, a1, a2, a3, a4, a5, a6)
 stmt_ty _Py_With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset,
-                 PyArena *arena);
-#define AsyncWith(a0, a1, a2, a3, a4) _Py_AsyncWith(a0, a1, a2, a3, a4)
+                 int end_lineno, int end_col_offset, PyArena *arena);
+#define AsyncWith(a0, a1, a2, a3, a4, a5, a6) _Py_AsyncWith(a0, a1, a2, a3, a4, a5, a6)
 stmt_ty _Py_AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int
-                      col_offset, PyArena *arena);
-#define Raise(a0, a1, a2, a3, a4) _Py_Raise(a0, a1, a2, a3, a4)
-stmt_ty _Py_Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset,
-                  PyArena *arena);
-#define Try(a0, a1, a2, a3, a4, a5, a6) _Py_Try(a0, a1, a2, a3, a4, a5, a6)
+                      col_offset, int end_lineno, int end_col_offset, PyArena
+                      *arena);
+#define Raise(a0, a1, a2, a3, a4, a5, a6) _Py_Raise(a0, a1, a2, a3, a4, a5, a6)
+stmt_ty _Py_Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, int
+                  end_lineno, int end_col_offset, PyArena *arena);
+#define Try(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_Try(a0, a1, a2, a3, a4, a5, a6, a7, a8)
 stmt_ty _Py_Try(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse,
-                asdl_seq * finalbody, int lineno, int col_offset, PyArena
-                *arena);
-#define Assert(a0, a1, a2, a3, a4) _Py_Assert(a0, a1, a2, a3, a4)
-stmt_ty _Py_Assert(expr_ty test, expr_ty msg, int lineno, int col_offset,
-                   PyArena *arena);
-#define Import(a0, a1, a2, a3) _Py_Import(a0, a1, a2, a3)
-stmt_ty _Py_Import(asdl_seq * names, int lineno, int col_offset, PyArena
-                   *arena);
-#define ImportFrom(a0, a1, a2, a3, a4, a5) _Py_ImportFrom(a0, a1, a2, a3, a4, a5)
+                asdl_seq * finalbody, int lineno, int col_offset, int
+                end_lineno, int end_col_offset, PyArena *arena);
+#define Assert(a0, a1, a2, a3, a4, a5, a6) _Py_Assert(a0, a1, a2, a3, a4, a5, a6)
+stmt_ty _Py_Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, int
+                   end_lineno, int end_col_offset, PyArena *arena);
+#define Import(a0, a1, a2, a3, a4, a5) _Py_Import(a0, a1, a2, a3, a4, a5)
+stmt_ty _Py_Import(asdl_seq * names, int lineno, int col_offset, int
+                   end_lineno, int end_col_offset, PyArena *arena);
+#define ImportFrom(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ImportFrom(a0, a1, a2, a3, a4, a5, a6, a7)
 stmt_ty _Py_ImportFrom(identifier module, asdl_seq * names, int level, int
-                       lineno, int col_offset, PyArena *arena);
-#define Global(a0, a1, a2, a3) _Py_Global(a0, a1, a2, a3)
-stmt_ty _Py_Global(asdl_seq * names, int lineno, int col_offset, PyArena
-                   *arena);
-#define Nonlocal(a0, a1, a2, a3) _Py_Nonlocal(a0, a1, a2, a3)
-stmt_ty _Py_Nonlocal(asdl_seq * names, int lineno, int col_offset, PyArena
-                     *arena);
-#define Expr(a0, a1, a2, a3) _Py_Expr(a0, a1, a2, a3)
-stmt_ty _Py_Expr(expr_ty value, int lineno, int col_offset, PyArena *arena);
-#define Pass(a0, a1, a2) _Py_Pass(a0, a1, a2)
-stmt_ty _Py_Pass(int lineno, int col_offset, PyArena *arena);
-#define Break(a0, a1, a2) _Py_Break(a0, a1, a2)
-stmt_ty _Py_Break(int lineno, int col_offset, PyArena *arena);
-#define Continue(a0, a1, a2) _Py_Continue(a0, a1, a2)
-stmt_ty _Py_Continue(int lineno, int col_offset, PyArena *arena);
-#define BoolOp(a0, a1, a2, a3, a4) _Py_BoolOp(a0, a1, a2, a3, a4)
+                       lineno, int col_offset, int end_lineno, int
+                       end_col_offset, PyArena *arena);
+#define Global(a0, a1, a2, a3, a4, a5) _Py_Global(a0, a1, a2, a3, a4, a5)
+stmt_ty _Py_Global(asdl_seq * names, int lineno, int col_offset, int
+                   end_lineno, int end_col_offset, PyArena *arena);
+#define Nonlocal(a0, a1, a2, a3, a4, a5) _Py_Nonlocal(a0, a1, a2, a3, a4, a5)
+stmt_ty _Py_Nonlocal(asdl_seq * names, int lineno, int col_offset, int
+                     end_lineno, int end_col_offset, PyArena *arena);
+#define Expr(a0, a1, a2, a3, a4, a5) _Py_Expr(a0, a1, a2, a3, a4, a5)
+stmt_ty _Py_Expr(expr_ty value, int lineno, int col_offset, int end_lineno, int
+                 end_col_offset, PyArena *arena);
+#define Pass(a0, a1, a2, a3, a4) _Py_Pass(a0, a1, a2, a3, a4)
+stmt_ty _Py_Pass(int lineno, int col_offset, int end_lineno, int
+                 end_col_offset, PyArena *arena);
+#define Break(a0, a1, a2, a3, a4) _Py_Break(a0, a1, a2, a3, a4)
+stmt_ty _Py_Break(int lineno, int col_offset, int end_lineno, int
+                  end_col_offset, PyArena *arena);
+#define Continue(a0, a1, a2, a3, a4) _Py_Continue(a0, a1, a2, a3, a4)
+stmt_ty _Py_Continue(int lineno, int col_offset, int end_lineno, int
+                     end_col_offset, PyArena *arena);
+#define BoolOp(a0, a1, a2, a3, a4, a5, a6) _Py_BoolOp(a0, a1, a2, a3, a4, a5, a6)
 expr_ty _Py_BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset,
-                   PyArena *arena);
-#define BinOp(a0, a1, a2, a3, a4, a5) _Py_BinOp(a0, a1, a2, a3, a4, a5)
+                   int end_lineno, int end_col_offset, PyArena *arena);
+#define BinOp(a0, a1, a2, a3, a4, a5, a6, a7) _Py_BinOp(a0, a1, a2, a3, a4, a5, a6, a7)
 expr_ty _Py_BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int
-                  col_offset, PyArena *arena);
-#define UnaryOp(a0, a1, a2, a3, a4) _Py_UnaryOp(a0, a1, a2, a3, a4)
+                  col_offset, int end_lineno, int end_col_offset, PyArena
+                  *arena);
+#define UnaryOp(a0, a1, a2, a3, a4, a5, a6) _Py_UnaryOp(a0, a1, a2, a3, a4, a5, a6)
 expr_ty _Py_UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset,
-                    PyArena *arena);
-#define Lambda(a0, a1, a2, a3, a4) _Py_Lambda(a0, a1, a2, a3, a4)
+                    int end_lineno, int end_col_offset, PyArena *arena);
+#define Lambda(a0, a1, a2, a3, a4, a5, a6) _Py_Lambda(a0, a1, a2, a3, a4, a5, a6)
 expr_ty _Py_Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset,
-                   PyArena *arena);
-#define IfExp(a0, a1, a2, a3, a4, a5) _Py_IfExp(a0, a1, a2, a3, a4, a5)
+                   int end_lineno, int end_col_offset, PyArena *arena);
+#define IfExp(a0, a1, a2, a3, a4, a5, a6, a7) _Py_IfExp(a0, a1, a2, a3, a4, a5, a6, a7)
 expr_ty _Py_IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int
-                  col_offset, PyArena *arena);
-#define Dict(a0, a1, a2, a3, a4) _Py_Dict(a0, a1, a2, a3, a4)
+                  col_offset, int end_lineno, int end_col_offset, PyArena
+                  *arena);
+#define Dict(a0, a1, a2, a3, a4, a5, a6) _Py_Dict(a0, a1, a2, a3, a4, a5, a6)
 expr_ty _Py_Dict(asdl_seq * keys, asdl_seq * values, int lineno, int
-                 col_offset, PyArena *arena);
-#define Set(a0, a1, a2, a3) _Py_Set(a0, a1, a2, a3)
-expr_ty _Py_Set(asdl_seq * elts, int lineno, int col_offset, PyArena *arena);
-#define ListComp(a0, a1, a2, a3, a4) _Py_ListComp(a0, a1, a2, a3, a4)
+                 col_offset, int end_lineno, int end_col_offset, PyArena
+                 *arena);
+#define Set(a0, a1, a2, a3, a4, a5) _Py_Set(a0, a1, a2, a3, a4, a5)
+expr_ty _Py_Set(asdl_seq * elts, int lineno, int col_offset, int end_lineno,
+                int end_col_offset, PyArena *arena);
+#define ListComp(a0, a1, a2, a3, a4, a5, a6) _Py_ListComp(a0, a1, a2, a3, a4, a5, a6)
 expr_ty _Py_ListComp(expr_ty elt, asdl_seq * generators, int lineno, int
-                     col_offset, PyArena *arena);
-#define SetComp(a0, a1, a2, a3, a4) _Py_SetComp(a0, a1, a2, a3, a4)
-expr_ty _Py_SetComp(expr_ty elt, asdl_seq * generators, int lineno, int
-                    col_offset, PyArena *arena);
-#define DictComp(a0, a1, a2, a3, a4, a5) _Py_DictComp(a0, a1, a2, a3, a4, a5)
-expr_ty _Py_DictComp(expr_ty key, expr_ty value, asdl_seq * generators, int
-                     lineno, int col_offset, PyArena *arena);
-#define GeneratorExp(a0, a1, a2, a3, a4) _Py_GeneratorExp(a0, a1, a2, a3, a4)
-expr_ty _Py_GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int
-                         col_offset, PyArena *arena);
-#define Await(a0, a1, a2, a3) _Py_Await(a0, a1, a2, a3)
-expr_ty _Py_Await(expr_ty value, int lineno, int col_offset, PyArena *arena);
-#define Yield(a0, a1, a2, a3) _Py_Yield(a0, a1, a2, a3)
-expr_ty _Py_Yield(expr_ty value, int lineno, int col_offset, PyArena *arena);
-#define YieldFrom(a0, a1, a2, a3) _Py_YieldFrom(a0, a1, a2, a3)
-expr_ty _Py_YieldFrom(expr_ty value, int lineno, int col_offset, PyArena
-                      *arena);
-#define Compare(a0, a1, a2, a3, a4, a5) _Py_Compare(a0, a1, a2, a3, a4, a5)
-expr_ty _Py_Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators,
-                    int lineno, int col_offset, PyArena *arena);
-#define Call(a0, a1, a2, a3, a4, a5) _Py_Call(a0, a1, a2, a3, a4, a5)
-expr_ty _Py_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int
-                 lineno, int col_offset, PyArena *arena);
-#define FormattedValue(a0, a1, a2, a3, a4, a5) _Py_FormattedValue(a0, a1, a2, a3, a4, a5)
-expr_ty _Py_FormattedValue(expr_ty value, int conversion, expr_ty format_spec,
-                           int lineno, int col_offset, PyArena *arena);
-#define JoinedStr(a0, a1, a2, a3) _Py_JoinedStr(a0, a1, a2, a3)
-expr_ty _Py_JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena
-                      *arena);
-#define Constant(a0, a1, a2, a3) _Py_Constant(a0, a1, a2, a3)
-expr_ty _Py_Constant(constant value, int lineno, int col_offset, PyArena
+                     col_offset, int end_lineno, int end_col_offset, PyArena
                      *arena);
-#define Attribute(a0, a1, a2, a3, a4, a5) _Py_Attribute(a0, a1, a2, a3, a4, a5)
+#define SetComp(a0, a1, a2, a3, a4, a5, a6) _Py_SetComp(a0, a1, a2, a3, a4, a5, a6)
+expr_ty _Py_SetComp(expr_ty elt, asdl_seq * generators, int lineno, int
+                    col_offset, int end_lineno, int end_col_offset, PyArena
+                    *arena);
+#define DictComp(a0, a1, a2, a3, a4, a5, a6, a7) _Py_DictComp(a0, a1, a2, a3, a4, a5, a6, a7)
+expr_ty _Py_DictComp(expr_ty key, expr_ty value, asdl_seq * generators, int
+                     lineno, int col_offset, int end_lineno, int
+                     end_col_offset, PyArena *arena);
+#define GeneratorExp(a0, a1, a2, a3, a4, a5, a6) _Py_GeneratorExp(a0, a1, a2, a3, a4, a5, a6)
+expr_ty _Py_GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int
+                         col_offset, int end_lineno, int end_col_offset,
+                         PyArena *arena);
+#define Await(a0, a1, a2, a3, a4, a5) _Py_Await(a0, a1, a2, a3, a4, a5)
+expr_ty _Py_Await(expr_ty value, int lineno, int col_offset, int end_lineno,
+                  int end_col_offset, PyArena *arena);
+#define Yield(a0, a1, a2, a3, a4, a5) _Py_Yield(a0, a1, a2, a3, a4, a5)
+expr_ty _Py_Yield(expr_ty value, int lineno, int col_offset, int end_lineno,
+                  int end_col_offset, PyArena *arena);
+#define YieldFrom(a0, a1, a2, a3, a4, a5) _Py_YieldFrom(a0, a1, a2, a3, a4, a5)
+expr_ty _Py_YieldFrom(expr_ty value, int lineno, int col_offset, int
+                      end_lineno, int end_col_offset, PyArena *arena);
+#define Compare(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Compare(a0, a1, a2, a3, a4, a5, a6, a7)
+expr_ty _Py_Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators,
+                    int lineno, int col_offset, int end_lineno, int
+                    end_col_offset, PyArena *arena);
+#define Call(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Call(a0, a1, a2, a3, a4, a5, a6, a7)
+expr_ty _Py_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int
+                 lineno, int col_offset, int end_lineno, int end_col_offset,
+                 PyArena *arena);
+#define FormattedValue(a0, a1, a2, a3, a4, a5, a6, a7) _Py_FormattedValue(a0, a1, a2, a3, a4, a5, a6, a7)
+expr_ty _Py_FormattedValue(expr_ty value, int conversion, expr_ty format_spec,
+                           int lineno, int col_offset, int end_lineno, int
+                           end_col_offset, PyArena *arena);
+#define JoinedStr(a0, a1, a2, a3, a4, a5) _Py_JoinedStr(a0, a1, a2, a3, a4, a5)
+expr_ty _Py_JoinedStr(asdl_seq * values, int lineno, int col_offset, int
+                      end_lineno, int end_col_offset, PyArena *arena);
+#define Constant(a0, a1, a2, a3, a4, a5) _Py_Constant(a0, a1, a2, a3, a4, a5)
+expr_ty _Py_Constant(constant value, int lineno, int col_offset, int
+                     end_lineno, int end_col_offset, PyArena *arena);
+#define Attribute(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Attribute(a0, a1, a2, a3, a4, a5, a6, a7)
 expr_ty _Py_Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int
-                      lineno, int col_offset, PyArena *arena);
-#define Subscript(a0, a1, a2, a3, a4, a5) _Py_Subscript(a0, a1, a2, a3, a4, a5)
+                      lineno, int col_offset, int end_lineno, int
+                      end_col_offset, PyArena *arena);
+#define Subscript(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Subscript(a0, a1, a2, a3, a4, a5, a6, a7)
 expr_ty _Py_Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int
-                      lineno, int col_offset, PyArena *arena);
-#define Starred(a0, a1, a2, a3, a4) _Py_Starred(a0, a1, a2, a3, a4)
+                      lineno, int col_offset, int end_lineno, int
+                      end_col_offset, PyArena *arena);
+#define Starred(a0, a1, a2, a3, a4, a5, a6) _Py_Starred(a0, a1, a2, a3, a4, a5, a6)
 expr_ty _Py_Starred(expr_ty value, expr_context_ty ctx, int lineno, int
-                    col_offset, PyArena *arena);
-#define Name(a0, a1, a2, a3, a4) _Py_Name(a0, a1, a2, a3, a4)
+                    col_offset, int end_lineno, int end_col_offset, PyArena
+                    *arena);
+#define Name(a0, a1, a2, a3, a4, a5, a6) _Py_Name(a0, a1, a2, a3, a4, a5, a6)
 expr_ty _Py_Name(identifier id, expr_context_ty ctx, int lineno, int
-                 col_offset, PyArena *arena);
-#define List(a0, a1, a2, a3, a4) _Py_List(a0, a1, a2, a3, a4)
+                 col_offset, int end_lineno, int end_col_offset, PyArena
+                 *arena);
+#define List(a0, a1, a2, a3, a4, a5, a6) _Py_List(a0, a1, a2, a3, a4, a5, a6)
 expr_ty _Py_List(asdl_seq * elts, expr_context_ty ctx, int lineno, int
-                 col_offset, PyArena *arena);
-#define Tuple(a0, a1, a2, a3, a4) _Py_Tuple(a0, a1, a2, a3, a4)
+                 col_offset, int end_lineno, int end_col_offset, PyArena
+                 *arena);
+#define Tuple(a0, a1, a2, a3, a4, a5, a6) _Py_Tuple(a0, a1, a2, a3, a4, a5, a6)
 expr_ty _Py_Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int
-                  col_offset, PyArena *arena);
+                  col_offset, int end_lineno, int end_col_offset, PyArena
+                  *arena);
 #define Slice(a0, a1, a2, a3) _Py_Slice(a0, a1, a2, a3)
 slice_ty _Py_Slice(expr_ty lower, expr_ty upper, expr_ty step, PyArena *arena);
 #define ExtSlice(a0, a1) _Py_ExtSlice(a0, a1)
@@ -593,17 +638,18 @@ slice_ty _Py_Index(expr_ty value, PyArena *arena);
 #define comprehension(a0, a1, a2, a3, a4) _Py_comprehension(a0, a1, a2, a3, a4)
 comprehension_ty _Py_comprehension(expr_ty target, expr_ty iter, asdl_seq *
                                    ifs, int is_async, PyArena *arena);
-#define ExceptHandler(a0, a1, a2, a3, a4, a5) _Py_ExceptHandler(a0, a1, a2, a3, a4, a5)
+#define ExceptHandler(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ExceptHandler(a0, a1, a2, a3, a4, a5, a6, a7)
 excepthandler_ty _Py_ExceptHandler(expr_ty type, identifier name, asdl_seq *
-                                   body, int lineno, int col_offset, PyArena
+                                   body, int lineno, int col_offset, int
+                                   end_lineno, int end_col_offset, PyArena
                                    *arena);
 #define arguments(a0, a1, a2, a3, a4, a5, a6) _Py_arguments(a0, a1, a2, a3, a4, a5, a6)
 arguments_ty _Py_arguments(asdl_seq * args, arg_ty vararg, asdl_seq *
                            kwonlyargs, asdl_seq * kw_defaults, arg_ty kwarg,
                            asdl_seq * defaults, PyArena *arena);
-#define arg(a0, a1, a2, a3, a4) _Py_arg(a0, a1, a2, a3, a4)
+#define arg(a0, a1, a2, a3, a4, a5, a6) _Py_arg(a0, a1, a2, a3, a4, a5, a6)
 arg_ty _Py_arg(identifier arg, expr_ty annotation, int lineno, int col_offset,
-               PyArena *arena);
+               int end_lineno, int end_col_offset, PyArena *arena);
 #define keyword(a0, a1, a2) _Py_keyword(a0, a1, a2)
 keyword_ty _Py_keyword(identifier arg, expr_ty value, PyArena *arena);
 #define alias(a0, a1, a2) _Py_alias(a0, a1, a2)

Include/node.h

@@ -14,11 +14,14 @@ typedef struct _node {
     int                 n_col_offset;
     int                 n_nchildren;
     struct _node        *n_child;
+    int                 n_end_lineno;
+    int                 n_end_col_offset;
 } node;
 
 PyAPI_FUNC(node *) PyNode_New(int type);
 PyAPI_FUNC(int) PyNode_AddChild(node *n, int type,
-                                      char *str, int lineno, int col_offset);
+                                char *str, int lineno, int col_offset,
+                                int end_lineno, int end_col_offset);
 PyAPI_FUNC(void) PyNode_Free(node *n);
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(Py_ssize_t) _PyNode_SizeOf(node *n);
@@ -37,6 +40,7 @@ PyAPI_FUNC(Py_ssize_t) _PyNode_SizeOf(node *n);
 #define REQ(n, type) assert(TYPE(n) == (type))
 
 PyAPI_FUNC(void) PyNode_ListTree(node *);
+void _PyNode_FinalizeEndPos(node *n);  // helper also used in parsetok.c
 
 #ifdef __cplusplus
 }

Lib/ast.py

@@ -115,10 +115,10 @@ def _format(node):
 
 def copy_location(new_node, old_node):
     """
-    Copy source location (`lineno` and `col_offset` attributes) from
-    *old_node* to *new_node* if possible, and return *new_node*.
+    Copy source location (`lineno`, `col_offset`, `end_lineno`, and `end_col_offset`
+    attributes) from *old_node* to *new_node* if possible, and return *new_node*.
     """
-    for attr in 'lineno', 'col_offset':
+    for attr in 'lineno', 'col_offset', 'end_lineno', 'end_col_offset':
         if attr in old_node._attributes and attr in new_node._attributes \
            and hasattr(old_node, attr):
             setattr(new_node, attr, getattr(old_node, attr))
@@ -133,31 +133,44 @@ def fix_missing_locations(node):
     recursively where not already set, by setting them to the values of the
     parent node.  It works recursively starting at *node*.
     """
-    def _fix(node, lineno, col_offset):
+    def _fix(node, lineno, col_offset, end_lineno, end_col_offset):
         if 'lineno' in node._attributes:
             if not hasattr(node, 'lineno'):
                 node.lineno = lineno
             else:
                 lineno = node.lineno
+        if 'end_lineno' in node._attributes:
+            if not hasattr(node, 'end_lineno'):
+                node.end_lineno = end_lineno
+            else:
+                end_lineno = node.end_lineno
         if 'col_offset' in node._attributes:
             if not hasattr(node, 'col_offset'):
                 node.col_offset = col_offset
             else:
                 col_offset = node.col_offset
+        if 'end_col_offset' in node._attributes:
+            if not hasattr(node, 'end_col_offset'):
+                node.end_col_offset = end_col_offset
+            else:
+                end_col_offset = node.end_col_offset
         for child in iter_child_nodes(node):
-            _fix(child, lineno, col_offset)
-    _fix(node, 1, 0)
+            _fix(child, lineno, col_offset, end_lineno, end_col_offset)
+    _fix(node, 1, 0, 1, 0)
     return node
 
 
 def increment_lineno(node, n=1):
     """
-    Increment the line number of each node in the tree starting at *node* by *n*.
-    This is useful to "move code" to a different location in a file.
+    Increment the line number and end line number of each node in the tree
+    starting at *node* by *n*. This is useful to "move code" to a different
+    location in a file.
     """
     for child in walk(node):
         if 'lineno' in child._attributes:
             child.lineno = getattr(child, 'lineno', 0) + n
+        if 'end_lineno' in child._attributes:
+            child.end_lineno = getattr(child, 'end_lineno', 0) + n
     return node
 
 
@@ -213,6 +226,77 @@ def get_docstring(node, clean=True):
     return text
 
 
+def _splitlines_no_ff(source):
+    """Split a string into lines ignoring form feed and other chars.
+
+    This mimics how the Python parser splits source code.
+    """
+    idx = 0
+    lines = []
+    next_line = ''
+    while idx < len(source):
+        c = source[idx]
+        next_line += c
+        idx += 1
+        # Keep \r\n together
+        if c == '\r' and idx < len(source) and source[idx] == '\n':
+            next_line += '\n'
+            idx += 1
+        if c in '\r\n':
+            lines.append(next_line)
+            next_line = ''
+
+    if next_line:
+        lines.append(next_line)
+    return lines
+
+
+def _pad_whitespace(source):
+    """Replace all chars except '\f\t' in a line with spaces."""
+    result = ''
+    for c in source:
+        if c in '\f\t':
+            result += c
+        else:
+            result += ' '
+    return result
+
+
+def get_source_segment(source, node, *, padded=False):
+    """Get source code segment of the *source* that generated *node*.
+
+    If some location information (`lineno`, `end_lineno`, `col_offset`,
+    or `end_col_offset`) is missing, return None.
+
+    If *padded* is `True`, the first line of a multi-line statement will
+    be padded with spaces to match its original position.
+    """
+    try:
+        lineno = node.lineno - 1
+        end_lineno = node.end_lineno - 1
+        col_offset = node.col_offset
+        end_col_offset = node.end_col_offset
+    except AttributeError:
+        return None
+
+    lines = _splitlines_no_ff(source)
+    if end_lineno == lineno:
+        return lines[lineno].encode()[col_offset:end_col_offset].decode()
+
+    if padded:
+        padding = _pad_whitespace(lines[lineno].encode()[:col_offset].decode())
+    else:
+        padding = ''
+
+    first = padding + lines[lineno].encode()[col_offset:].decode()
+    last = lines[end_lineno].encode()[:end_col_offset].decode()
+    lines = lines[lineno+1:end_lineno]
+
+    lines.insert(0, first)
+    lines.append(last)
+    return ''.join(lines)
+
+
 def walk(node):
     """
     Recursively yield all descendant nodes in the tree starting at *node*

Lib/test/test_asdl_parser.py

@@ -62,14 +62,16 @@ def test_product(self):
 
     def test_attributes(self):
         stmt = self.types['stmt']
-        self.assertEqual(len(stmt.attributes), 2)
+        self.assertEqual(len(stmt.attributes), 4)
         self.assertEqual(str(stmt.attributes[0]), 'Field(int, lineno)')
         self.assertEqual(str(stmt.attributes[1]), 'Field(int, col_offset)')
+        self.assertEqual(str(stmt.attributes[2]), 'Field(int, end_lineno, opt=True)')
+        self.assertEqual(str(stmt.attributes[3]), 'Field(int, end_col_offset, opt=True)')
 
     def test_constructor_fields(self):
         ehandler = self.types['excepthandler']
         self.assertEqual(len(ehandler.types), 1)
-        self.assertEqual(len(ehandler.attributes), 2)
+        self.assertEqual(len(ehandler.attributes), 4)
 
         cons = ehandler.types[0]
         self.assertIsInstance(cons, self.asdl.Constructor)

Lib/test/test_ast.py

@@ -4,6 +4,7 @@
 import sys
 import unittest
 import weakref
+from textwrap import dedent
 
 from test import support
 
@@ -582,19 +583,22 @@ def test_dump(self):
         )
         self.assertEqual(ast.dump(node, include_attributes=True),
             "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), "
-            "lineno=1, col_offset=0), args=[Name(id='eggs', ctx=Load(), "
-            "lineno=1, col_offset=5), Constant(value='and cheese', lineno=1, "
-            "col_offset=11)], keywords=[], "
-            "lineno=1, col_offset=0), lineno=1, col_offset=0)])"
+            "lineno=1, col_offset=0, end_lineno=1, end_col_offset=4), "
+            "args=[Name(id='eggs', ctx=Load(), lineno=1, col_offset=5, "
+            "end_lineno=1, end_col_offset=9), Constant(value='and cheese', "
+            "lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], keywords=[], "
+            "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24), "
+            "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)])"
         )
 
     def test_copy_location(self):
         src = ast.parse('1 + 1', mode='eval')
         src.body.right = ast.copy_location(ast.Num(2), src.body.right)
         self.assertEqual(ast.dump(src, include_attributes=True),
-            'Expression(body=BinOp(left=Constant(value=1, lineno=1, col_offset=0), '
-            'op=Add(), right=Constant(value=2, lineno=1, col_offset=4), lineno=1, '
-            'col_offset=0))'
+            'Expression(body=BinOp(left=Constant(value=1, lineno=1, col_offset=0, '
+            'end_lineno=1, end_col_offset=1), op=Add(), right=Constant(value=2, '
+            'lineno=1, col_offset=4, end_lineno=1, end_col_offset=5), lineno=1, '
+            'col_offset=0, end_lineno=1, end_col_offset=5))'
         )
 
     def test_fix_missing_locations(self):
@@ -602,32 +606,37 @@ def test_fix_missing_locations(self):
         src.body.append(ast.Expr(ast.Call(ast.Name('spam', ast.Load()),
                                           [ast.Str('eggs')], [])))
         self.assertEqual(src, ast.fix_missing_locations(src))
+        self.maxDiff = None
         self.assertEqual(ast.dump(src, include_attributes=True),
             "Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), "
-            "lineno=1, col_offset=0), args=[Constant(value='spam', lineno=1, "
-            "col_offset=6)], keywords=[], "
-            "lineno=1, col_offset=0), lineno=1, col_offset=0), "
-            "Expr(value=Call(func=Name(id='spam', ctx=Load(), lineno=1, "
-            "col_offset=0), args=[Constant(value='eggs', lineno=1, col_offset=0)], "
-            "keywords=[], lineno=1, "
-            "col_offset=0), lineno=1, col_offset=0)])"
+            "lineno=1, col_offset=0, end_lineno=1, end_col_offset=5), "
+            "args=[Constant(value='spam', lineno=1, col_offset=6, end_lineno=1, "
+            "end_col_offset=12)], keywords=[], lineno=1, col_offset=0, end_lineno=1, "
+            "end_col_offset=13), lineno=1, col_offset=0, end_lineno=1, "
+            "end_col_offset=13), Expr(value=Call(func=Name(id='spam', ctx=Load(), "
+            "lineno=1, col_offset=0, end_lineno=1, end_col_offset=0), "
+            "args=[Constant(value='eggs', lineno=1, col_offset=0, end_lineno=1, "
+            "end_col_offset=0)], keywords=[], lineno=1, col_offset=0, end_lineno=1, "
+            "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)])"
         )
 
     def test_increment_lineno(self):
         src = ast.parse('1 + 1', mode='eval')
         self.assertEqual(ast.increment_lineno(src, n=3), src)
         self.assertEqual(ast.dump(src, include_attributes=True),
-            'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0), '
-            'op=Add(), right=Constant(value=1, lineno=4, col_offset=4), lineno=4, '
-            'col_offset=0))'
+            'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, '
+            'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, '
+            'lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, '
+            'col_offset=0, end_lineno=4, end_col_offset=5))'
         )
         # issue10869: do not increment lineno of root twice
         src = ast.parse('1 + 1', mode='eval')
         self.assertEqual(ast.increment_lineno(src.body, n=3), src.body)
         self.assertEqual(ast.dump(src, include_attributes=True),
-            'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0), '
-            'op=Add(), right=Constant(value=1, lineno=4, col_offset=4), lineno=4, '
-            'col_offset=0))'
+            'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, '
+            'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, '
+            'lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, '
+            'col_offset=0, end_lineno=4, end_col_offset=5))'
         )
 
     def test_iter_fields(self):
@@ -1274,6 +1283,311 @@ def test_literal_eval(self):
         self.assertEqual(ast.literal_eval(binop), 10+20j)
 
 
+class EndPositionTests(unittest.TestCase):
+    """Tests for end position of AST nodes.
+
+    Testing end positions of nodes requires a bit of extra care
+    because of how LL parsers work.
+    """
+    def _check_end_pos(self, ast_node, end_lineno, end_col_offset):
+        self.assertEqual(ast_node.end_lineno, end_lineno)
+        self.assertEqual(ast_node.end_col_offset, end_col_offset)
+
+    def _check_content(self, source, ast_node, content):
+        self.assertEqual(ast.get_source_segment(source, ast_node), content)
+
+    def _parse_value(self, s):
+        # Use duck-typing to support both single expression
+        # and a right hand side of an assignment statement.
+        return ast.parse(s).body[0].value
+
+    def test_lambda(self):
+        s = 'lambda x, *y: None'
+        lam = self._parse_value(s)
+        self._check_content(s, lam.body, 'None')
+        self._check_content(s, lam.args.args[0], 'x')
+        self._check_content(s, lam.args.vararg, 'y')
+
+    def test_func_def(self):
+        s = dedent('''
+            def func(x: int,
+                     *args: str,
+                     z: float = 0,
+                     **kwargs: Any) -> bool:
+                return True
+            ''').strip()
+        fdef = ast.parse(s).body[0]
+        self._check_end_pos(fdef, 5, 15)
+        self._check_content(s, fdef.body[0], 'return True')
+        self._check_content(s, fdef.args.args[0], 'x: int')
+        self._check_content(s, fdef.args.args[0].annotation, 'int')
+        self._check_content(s, fdef.args.kwarg, 'kwargs: Any')
+        self._check_content(s, fdef.args.kwarg.annotation, 'Any')
+
+    def test_call(self):
+        s = 'func(x, y=2, **kw)'
+        call = self._parse_value(s)
+        self._check_content(s, call.func, 'func')
+        self._check_content(s, call.keywords[0].value, '2')
+        self._check_content(s, call.keywords[1].value, 'kw')
+
+    def test_call_noargs(self):
+        s = 'x[0]()'
+        call = self._parse_value(s)
+        self._check_content(s, call.func, 'x[0]')
+        self._check_end_pos(call, 1, 6)
+
+    def test_class_def(self):
+        s = dedent('''
+            class C(A, B):
+                x: int = 0
+        ''').strip()
+        cdef = ast.parse(s).body[0]
+        self._check_end_pos(cdef, 2, 14)
+        self._check_content(s, cdef.bases[1], 'B')
+        self._check_content(s, cdef.body[0], 'x: int = 0')
+
+    def test_class_kw(self):
+        s = 'class S(metaclass=abc.ABCMeta): pass'
+        cdef = ast.parse(s).body[0]
+        self._check_content(s, cdef.keywords[0].value, 'abc.ABCMeta')
+
+    def test_multi_line_str(self):
+        s = dedent('''
+            x = """Some multi-line text.
+
+            It goes on starting from same indent."""
+        ''').strip()
+        assign = ast.parse(s).body[0]
+        self._check_end_pos(assign, 3, 40)
+        self._check_end_pos(assign.value, 3, 40)
+
+    def test_continued_str(self):
+        s = dedent('''
+            x = "first part" \\
+            "second part"
+        ''').strip()
+        assign = ast.parse(s).body[0]
+        self._check_end_pos(assign, 2, 13)
+        self._check_end_pos(assign.value, 2, 13)
+
+    def test_suites(self):
+        # We intentionally put these into the same string to check
+        # that empty lines are not part of the suite.
+        s = dedent('''
+            while True:
+                pass
+
+            if one():
+                x = None
+            elif other():
+                y = None
+            else:
+                z = None
+
+            for x, y in stuff:
+                assert True
+
+            try:
+                raise RuntimeError
+            except TypeError as e:
+                pass
+
+            pass
+        ''').strip()
+        mod = ast.parse(s)
+        while_loop = mod.body[0]
+        if_stmt = mod.body[1]
+        for_loop = mod.body[2]
+        try_stmt = mod.body[3]
+        pass_stmt = mod.body[4]
+
+        self._check_end_pos(while_loop, 2, 8)
+        self._check_end_pos(if_stmt, 9, 12)
+        self._check_end_pos(for_loop, 12, 15)
+        self._check_end_pos(try_stmt, 17, 8)
+        self._check_end_pos(pass_stmt, 19, 4)
+
+        self._check_content(s, while_loop.test, 'True')
+        self._check_content(s, if_stmt.body[0], 'x = None')
+        self._check_content(s, if_stmt.orelse[0].test, 'other()')
+        self._check_content(s, for_loop.target, 'x, y')
+        self._check_content(s, try_stmt.body[0], 'raise RuntimeError')
+        self._check_content(s, try_stmt.handlers[0].type, 'TypeError')
+
+    def test_fstring(self):
+        s = 'x = f"abc {x + y} abc"'
+        fstr = self._parse_value(s)
+        binop = fstr.values[1].value
+        self._check_content(s, binop, 'x + y')
+
+    def test_fstring_multi_line(self):
+        s = dedent('''
+            f"""Some multi-line text.
+            {
+            arg_one
+            +
+            arg_two
+            }
+            It goes on..."""
+        ''').strip()
+        fstr = self._parse_value(s)
+        binop = fstr.values[1].value
+        self._check_end_pos(binop, 5, 7)
+        self._check_content(s, binop.left, 'arg_one')
+        self._check_content(s, binop.right, 'arg_two')
+
+    def test_import_from_multi_line(self):
+        s = dedent('''
+            from x.y.z import (
+                a, b, c as c
+            )
+        ''').strip()
+        imp = ast.parse(s).body[0]
+        self._check_end_pos(imp, 3, 1)
+
+    def test_slices(self):
+        s1 = 'f()[1, 2] [0]'
+        s2 = 'x[ a.b: c.d]'
+        sm = dedent('''
+            x[ a.b: f () ,
+               g () : c.d
+              ]
+        ''').strip()
+        i1, i2, im = map(self._parse_value, (s1, s2, sm))
+        self._check_content(s1, i1.value, 'f()[1, 2]')
+        self._check_content(s1, i1.value.slice.value, '1, 2')
+        self._check_content(s2, i2.slice.lower, 'a.b')
+        self._check_content(s2, i2.slice.upper, 'c.d')
+        self._check_content(sm, im.slice.dims[0].upper, 'f ()')
+        self._check_content(sm, im.slice.dims[1].lower, 'g ()')
+        self._check_end_pos(im, 3, 3)
+
+    def test_binop(self):
+        s = dedent('''
+            (1 * 2 + (3 ) +
+                 4
+            )
+        ''').strip()
+        binop = self._parse_value(s)
+        self._check_end_pos(binop, 2, 6)
+        self._check_content(s, binop.right, '4')
+        self._check_content(s, binop.left, '1 * 2 + (3 )')
+        self._check_content(s, binop.left.right, '3')
+
+    def test_boolop(self):
+        s = dedent('''
+            if (one_condition and
+                    (other_condition or yet_another_one)):
+                pass
+        ''').strip()
+        bop = ast.parse(s).body[0].test
+        self._check_end_pos(bop, 2, 44)
+        self._check_content(s, bop.values[1],
+                            'other_condition or yet_another_one')
+
+    def test_tuples(self):
+        s1 = 'x = () ;'
+        s2 = 'x = 1 , ;'
+        s3 = 'x = (1 , 2 ) ;'
+        sm = dedent('''
+            x = (
+                a, b,
+            )
+        ''').strip()
+        t1, t2, t3, tm = map(self._parse_value, (s1, s2, s3, sm))
+        self._check_content(s1, t1, '()')
+        self._check_content(s2, t2, '1 ,')
+        self._check_content(s3, t3, '(1 , 2 )')
+        self._check_end_pos(tm, 3, 1)
+
+    def test_attribute_spaces(self):
+        s = 'func(x. y .z)'
+        call = self._parse_value(s)
+        self._check_content(s, call, s)
+        self._check_content(s, call.args[0], 'x. y .z')
+
+    def test_displays(self):
+        s1 = '[{}, {1, }, {1, 2,} ]'
+        s2 = '{a: b, f (): g () ,}'
+        c1 = self._parse_value(s1)
+        c2 = self._parse_value(s2)
+        self._check_content(s1, c1.elts[0], '{}')
+        self._check_content(s1, c1.elts[1], '{1, }')
+        self._check_content(s1, c1.elts[2], '{1, 2,}')
+        self._check_content(s2, c2.keys[1], 'f ()')
+        self._check_content(s2, c2.values[1], 'g ()')
+
+    def test_comprehensions(self):
+        s = dedent('''
+            x = [{x for x, y in stuff
+                  if cond.x} for stuff in things]
+        ''').strip()
+        cmp = self._parse_value(s)
+        self._check_end_pos(cmp, 2, 37)
+        self._check_content(s, cmp.generators[0].iter, 'things')
+        self._check_content(s, cmp.elt.generators[0].iter, 'stuff')
+        self._check_content(s, cmp.elt.generators[0].ifs[0], 'cond.x')
+        self._check_content(s, cmp.elt.generators[0].target, 'x, y')
+
+    def test_yield_await(self):
+        s = dedent('''
+            async def f():
+                yield x
+                await y
+        ''').strip()
+        fdef = ast.parse(s).body[0]
+        self._check_content(s, fdef.body[0].value, 'yield x')
+        self._check_content(s, fdef.body[1].value, 'await y')
+
+    def test_source_segment_multi(self):
+        s_orig = dedent('''
+            x = (
+                a, b,
+            ) + ()
+        ''').strip()
+        s_tuple = dedent('''
+            (
+                a, b,
+            )
+        ''').strip()
+        binop = self._parse_value(s_orig)
+        self.assertEqual(ast.get_source_segment(s_orig, binop.left), s_tuple)
+
+    def test_source_segment_padded(self):
+        s_orig = dedent('''
+            class C:
+                def fun(self) -> None:
+                    "ЖЖЖЖЖ"
+        ''').strip()
+        s_method = '    def fun(self) -> None:\n' \
+                   '        "ЖЖЖЖЖ"'
+        cdef = ast.parse(s_orig).body[0]
+        self.assertEqual(ast.get_source_segment(s_orig, cdef.body[0], padded=True),
+                         s_method)
+
+    def test_source_segment_endings(self):
+        s = 'v = 1\r\nw = 1\nx = 1\n\ry = 1\rz = 1\r\n'
+        v, w, x, y, z = ast.parse(s).body
+        self._check_content(s, v, 'v = 1')
+        self._check_content(s, w, 'w = 1')
+        self._check_content(s, x, 'x = 1')
+        self._check_content(s, y, 'y = 1')
+        self._check_content(s, z, 'z = 1')
+
+    def test_source_segment_tabs(self):
+        s = dedent('''
+            class C:
+              \t\f  def fun(self) -> None:
+              \t\f      pass
+        ''').strip()
+        s_method = '  \t\f  def fun(self) -> None:\n' \
+                   '  \t\f      pass'
+
+        cdef = ast.parse(s).body[0]
+        self.assertEqual(ast.get_source_segment(s, cdef.body[0], padded=True), s_method)
+
+
 def main():
     if __name__ != '__main__':
         return

Lib/test/test_parser.py

@@ -880,7 +880,7 @@ def XXXROUNDUP(n):
             return 1 << (n - 1).bit_length()
 
         basesize = support.calcobjsize('Pii')
-        nodesize = struct.calcsize('hP3iP0h')
+        nodesize = struct.calcsize('hP3iP0h2i')
         def sizeofchildren(node):
             if node is None:
                 return 0

Misc/NEWS.d/next/Core and Builtins/2019-01-19-19-41-53.bpo-33416.VDeOU5.rst

@@ -0,0 +1,2 @@
+Add end line and end column position information to the Python AST nodes.
+This is a C-level backwards incompatible change.

Modules/parsermodule.c

@@ -920,7 +920,7 @@ build_node_children(PyObject *tuple, node *root, int *line_num)
             Py_DECREF(elem);
             return NULL;
         }
-        err = PyNode_AddChild(root, type, strn, *line_num, 0);
+        err = PyNode_AddChild(root, type, strn, *line_num, 0, *line_num, 0);
         if (err == E_NOMEM) {
             Py_DECREF(elem);
             PyObject_FREE(strn);

Parser/Python.asdl

@@ -50,7 +50,7 @@ module Python
 
           -- XXX Jython will be different
           -- col_offset is the byte offset in the utf8 string the parser uses
-          attributes (int lineno, int col_offset)
+          attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset)
 
           -- BoolOp() can use left & right?
     expr = BoolOp(boolop op, expr* values)
@@ -85,7 +85,7 @@ module Python
          | Tuple(expr* elts, expr_context ctx)
 
           -- col_offset is the byte offset in the utf8 string the parser uses
-          attributes (int lineno, int col_offset)
+          attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset)
 
     expr_context = Load | Store | Del | AugLoad | AugStore | Param
 
@@ -105,13 +105,13 @@ module Python
     comprehension = (expr target, expr iter, expr* ifs, int is_async)
 
     excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body)
-                    attributes (int lineno, int col_offset)
+                    attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset)
 
     arguments = (arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults,
                  arg? kwarg, expr* defaults)
 
     arg = (identifier arg, expr? annotation)
-           attributes (int lineno, int col_offset)
+           attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset)
 
     -- keyword arguments supplied to call (NULL identifier for **kwargs)
     keyword = (identifier? arg, expr value)

Parser/asdl_c.py

@@ -1250,10 +1250,12 @@ def main(srcfile, dump_module=False):
             f.write('#undef Yield   /* undefine macro conflicting with <winbase.h> */\n')
             f.write('\n')
             c = ChainOfVisitors(TypeDefVisitor(f),
-                                StructVisitor(f),
-                                PrototypeVisitor(f),
-                                )
+                                StructVisitor(f))
+
             c.visit(mod)
+            f.write("// Note: these macros affect function definitions, not only call sites.\n")
+            PrototypeVisitor(f).visit(mod)
+            f.write("\n")
             f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
             f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
             f.write("int PyAST_Check(PyObject* obj);\n")

Parser/node.c

@@ -13,6 +13,8 @@ PyNode_New(int type)
     n->n_type = type;
     n->n_str = NULL;
     n->n_lineno = 0;
+    n->n_end_lineno = 0;
+    n->n_end_col_offset = -1;
     n->n_nchildren = 0;
     n->n_child = NULL;
     return n;
@@ -75,14 +77,34 @@ fancy_roundup(int n)
                fancy_roundup(n))
 
 
+void
+_PyNode_FinalizeEndPos(node *n)
+{
+    int nch = NCH(n);
+    node *last;
+    if (nch == 0) {
+        return;
+    }
+    last = CHILD(n, nch - 1);
+    _PyNode_FinalizeEndPos(last);
+    n->n_end_lineno = last->n_end_lineno;
+    n->n_end_col_offset = last->n_end_col_offset;
+}
+
 int
-PyNode_AddChild(node *n1, int type, char *str, int lineno, int col_offset)
+PyNode_AddChild(node *n1, int type, char *str, int lineno, int col_offset,
+                int end_lineno, int end_col_offset)
 {
     const int nch = n1->n_nchildren;
     int current_capacity;
     int required_capacity;
     node *n;
 
+    // finalize end position of previous node (if any)
+    if (nch > 0) {
+        _PyNode_FinalizeEndPos(CHILD(n1, nch - 1));
+    }
+
     if (nch == INT_MAX || nch < 0)
         return E_OVERFLOW;
 
@@ -107,6 +129,8 @@ PyNode_AddChild(node *n1, int type, char *str, int lineno, int col_offset)
     n->n_str = str;
     n->n_lineno = lineno;
     n->n_col_offset = col_offset;
+    n->n_end_lineno = end_lineno;  // this and below will be updates after all children are added.
+    n->n_end_col_offset = end_col_offset;
     n->n_nchildren = 0;
     n->n_child = NULL;
     return 0;

Parser/parser.c

@@ -105,11 +105,13 @@ PyParser_Delete(parser_state *ps)
 /* PARSER STACK OPERATIONS */
 
 static int
-shift(stack *s, int type, char *str, int newstate, int lineno, int col_offset)
+shift(stack *s, int type, char *str, int newstate, int lineno, int col_offset,
+      int end_lineno, int end_col_offset)
 {
     int err;
     assert(!s_empty(s));
-    err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno, col_offset);
+    err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno, col_offset,
+                          end_lineno, end_col_offset);
     if (err)
         return err;
     s->s_top->s_state = newstate;
@@ -117,13 +119,15 @@ shift(stack *s, int type, char *str, int newstate, int lineno, int col_offset)
 }
 
 static int
-push(stack *s, int type, dfa *d, int newstate, int lineno, int col_offset)
+push(stack *s, int type, dfa *d, int newstate, int lineno, int col_offset,
+     int end_lineno, int end_col_offset)
 {
     int err;
     node *n;
     n = s->s_top->s_parent;
     assert(!s_empty(s));
-    err = PyNode_AddChild(n, type, (char *)NULL, lineno, col_offset);
+    err = PyNode_AddChild(n, type, (char *)NULL, lineno, col_offset,
+                          end_lineno, end_col_offset);
     if (err)
         return err;
     s->s_top->s_state = newstate;
@@ -225,7 +229,9 @@ future_hack(parser_state *ps)
 
 int
 PyParser_AddToken(parser_state *ps, int type, char *str,
-                  int lineno, int col_offset, int *expected_ret)
+                  int lineno, int col_offset,
+                  int end_lineno, int end_col_offset,
+                  int *expected_ret)
 {
     int ilabel;
     int err;
@@ -257,7 +263,8 @@ PyParser_AddToken(parser_state *ps, int type, char *str,
                     dfa *d1 = PyGrammar_FindDFA(
                         ps->p_grammar, nt);
                     if ((err = push(&ps->p_stack, nt, d1,
-                        arrow, lineno, col_offset)) > 0) {
+                        arrow, lineno, col_offset,
+                        end_lineno, end_col_offset)) > 0) {
                         D(printf(" MemError: push\n"));
                         return err;
                     }
@@ -267,7 +274,8 @@ PyParser_AddToken(parser_state *ps, int type, char *str,
 
                 /* Shift the token */
                 if ((err = shift(&ps->p_stack, type, str,
-                                x, lineno, col_offset)) > 0) {
+                                x, lineno, col_offset,
+                                end_lineno, end_col_offset)) > 0) {
                     D(printf(" MemError: shift.\n"));
                     return err;
                 }

Parser/parser.h

@@ -32,7 +32,9 @@ typedef struct {
 
 parser_state *PyParser_New(grammar *g, int start);
 void PyParser_Delete(parser_state *ps);
-int PyParser_AddToken(parser_state *ps, int type, char *str, int lineno, int col_offset,
+int PyParser_AddToken(parser_state *ps, int type, char *str,
+                      int lineno, int col_offset,
+                      int end_lineno, int end_col_offset,
                       int *expected_ret);
 void PyGrammar_AddAccelerators(grammar *g);
 

Parser/parsetok.c

@@ -187,7 +187,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
     parser_state *ps;
     node *n;
     int started = 0;
-    int col_offset;
+    int col_offset, end_col_offset;
 
     if ((ps = PyParser_New(g, start)) == NULL) {
         err_ret->error = E_NOMEM;
@@ -270,9 +270,16 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
             col_offset = -1;
         }
 
+        if (b != NULL && b >= tok->line_start) {
+            end_col_offset = Py_SAFE_DOWNCAST(b - tok->line_start,
+                                              intptr_t, int);
+        }
+        else {
+            end_col_offset = -1;
+        }
         if ((err_ret->error =
              PyParser_AddToken(ps, (int)type, str,
-                               lineno, col_offset,
+                               lineno, col_offset, tok->lineno, end_col_offset,
                                &(err_ret->expected))) != E_OK) {
             if (err_ret->error != E_DONE) {
                 PyObject_FREE(str);
@@ -368,6 +375,9 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
 done:
     PyTokenizer_Free(tok);
 
+    if (n != NULL) {
+        _PyNode_FinalizeEndPos(n);
+    }
     return n;
 }
 

Python/ast.c

@@ -580,10 +580,11 @@ static stmt_ty ast_for_for_stmt(struct compiling *, const node *, bool);
 
 /* Note different signature for ast_for_call */
 static expr_ty ast_for_call(struct compiling *, const node *, expr_ty,
-                            const node *);
+                            const node *, const node *);
 
 static PyObject *parsenumber(struct compiling *, const char *);
 static expr_ty parsestrplus(struct compiling *, const node *n);
+static void get_last_end_pos(asdl_seq *, int *, int *);
 
 #define COMP_GENEXP   0
 #define COMP_LISTCOMP 1
@@ -810,6 +811,7 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags,
                 if (!stmts)
                     goto out;
                 asdl_seq_SET(stmts, 0, Pass(n->n_lineno, n->n_col_offset,
+                                            n->n_end_lineno, n->n_end_col_offset,
                                             arena));
                 if (!asdl_seq_GET(stmts, 0))
                     goto out;
@@ -940,6 +942,8 @@ copy_location(expr_ty e, const node *n)
     if (e) {
         e->lineno = LINENO(n);
         e->col_offset = n->n_col_offset;
+        e->end_lineno = n->n_end_lineno;
+        e->end_col_offset = n->n_end_col_offset;
     }
     return e;
 }
@@ -1228,7 +1232,8 @@ ast_for_arg(struct compiling *c, const node *n)
             return NULL;
     }
 
-    ret = arg(name, annotation, LINENO(n), n->n_col_offset, c->c_arena);
+    ret = arg(name, annotation, LINENO(n), n->n_col_offset,
+              n->n_end_lineno, n->n_end_col_offset, c->c_arena);
     if (!ret)
         return NULL;
     return ret;
@@ -1287,6 +1292,7 @@ handle_keywordonly_args(struct compiling *c, const node *n, int start,
                 if (forbidden_name(c, argname, ch, 0))
                     goto error;
                 arg = arg(argname, annotation, LINENO(ch), ch->n_col_offset,
+                          ch->n_end_lineno, ch->n_end_col_offset,
                           c->c_arena);
                 if (!arg)
                     goto error;
@@ -1480,16 +1486,19 @@ ast_for_dotted_name(struct compiling *c, const node *n)
     identifier id;
     int lineno, col_offset;
     int i;
+    node *ch;
 
     REQ(n, dotted_name);
 
     lineno = LINENO(n);
     col_offset = n->n_col_offset;
 
-    id = NEW_IDENTIFIER(CHILD(n, 0));
+    ch = CHILD(n, 0);
+    id = NEW_IDENTIFIER(ch);
     if (!id)
         return NULL;
-    e = Name(id, Load, lineno, col_offset, c->c_arena);
+    e = Name(id, Load, lineno, col_offset,
+             ch->n_end_lineno, ch->n_end_col_offset, c->c_arena);
     if (!e)
         return NULL;
 
@@ -1497,7 +1506,8 @@ ast_for_dotted_name(struct compiling *c, const node *n)
         id = NEW_IDENTIFIER(CHILD(n, i));
         if (!id)
             return NULL;
-        e = Attribute(e, id, Load, lineno, col_offset, c->c_arena);
+        e = Attribute(e, id, Load, lineno, col_offset,
+                      n->n_end_lineno, n->n_end_col_offset, c->c_arena);
         if (!e)
             return NULL;
     }
@@ -1526,13 +1536,13 @@ ast_for_decorator(struct compiling *c, const node *n)
     }
     else if (NCH(n) == 5) { /* Call with no arguments */
         d = Call(name_expr, NULL, NULL, LINENO(n),
-                 n->n_col_offset, c->c_arena);
+                 n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena);
         if (!d)
             return NULL;
         name_expr = NULL;
     }
     else {
-        d = ast_for_call(c, CHILD(n, 3), name_expr, CHILD(n, 2));
+        d = ast_for_call(c, CHILD(n, 3), name_expr, CHILD(n, 2), CHILD(n, 4));
         if (!d)
             return NULL;
         name_expr = NULL;
@@ -1573,6 +1583,7 @@ ast_for_funcdef_impl(struct compiling *c, const node *n0,
     asdl_seq *body;
     expr_ty returns = NULL;
     int name_i = 1;
+    int end_lineno, end_col_offset;
 
     REQ(n, funcdef);
 
@@ -1593,13 +1604,14 @@ ast_for_funcdef_impl(struct compiling *c, const node *n0,
     body = ast_for_suite(c, CHILD(n, name_i + 3));
     if (!body)
         return NULL;
+    get_last_end_pos(body, &end_lineno, &end_col_offset);
 
     if (is_async)
         return AsyncFunctionDef(name, args, body, decorator_seq, returns,
-                                LINENO(n0), n0->n_col_offset, c->c_arena);
+                                LINENO(n0), n0->n_col_offset, end_lineno, end_col_offset, c->c_arena);
     else
         return FunctionDef(name, args, body, decorator_seq, returns,
-                           LINENO(n), n->n_col_offset, c->c_arena);
+                           LINENO(n), n->n_col_offset, end_lineno, end_col_offset, c->c_arena);
 }
 
 static stmt_ty
@@ -1704,7 +1716,8 @@ ast_for_lambdef(struct compiling *c, const node *n)
             return NULL;
     }
 
-    return Lambda(args, expression, LINENO(n), n->n_col_offset, c->c_arena);
+    return Lambda(args, expression, LINENO(n), n->n_col_offset,
+                  n->n_end_lineno, n->n_end_col_offset, c->c_arena);
 }
 
 static expr_ty
@@ -1724,6 +1737,7 @@ ast_for_ifexpr(struct compiling *c, const node *n)
     if (!orelse)
         return NULL;
     return IfExp(expression, body, orelse, LINENO(n), n->n_col_offset,
+                 n->n_end_lineno, n->n_end_col_offset,
                  c->c_arena);
 }
 
@@ -1852,8 +1866,9 @@ ast_for_comprehension(struct compiling *c, const node *n)
             comp = comprehension(first, expression, NULL,
                                  is_async, c->c_arena);
         else
-            comp = comprehension(Tuple(t, Store, first->lineno,
-                                       first->col_offset, c->c_arena),
+            comp = comprehension(Tuple(t, Store, first->lineno, first->col_offset,
+                                       for_ch->n_end_lineno, for_ch->n_end_col_offset,
+                                       c->c_arena),
                                  expression, NULL, is_async, c->c_arena);
         if (!comp)
             return NULL;
@@ -1918,11 +1933,14 @@ ast_for_itercomp(struct compiling *c, const node *n, int type)
         return NULL;
 
     if (type == COMP_GENEXP)
-        return GeneratorExp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena);
+        return GeneratorExp(elt, comps, LINENO(n), n->n_col_offset,
+                            n->n_end_lineno, n->n_end_col_offset, c->c_arena);
     else if (type == COMP_LISTCOMP)
-        return ListComp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena);
+        return ListComp(elt, comps, LINENO(n), n->n_col_offset,
+                        n->n_end_lineno, n->n_end_col_offset, c->c_arena);
     else if (type == COMP_SETCOMP)
-        return SetComp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena);
+        return SetComp(elt, comps, LINENO(n), n->n_col_offset,
+                       n->n_end_lineno, n->n_end_col_offset, c->c_arena);
     else
         /* Should never happen */
         return NULL;
@@ -1984,7 +2002,8 @@ ast_for_dictcomp(struct compiling *c, const node *n)
     if (!comps)
         return NULL;
 
-    return DictComp(key, value, comps, LINENO(n), n->n_col_offset, c->c_arena);
+    return DictComp(key, value, comps, LINENO(n), n->n_col_offset,
+                    n->n_end_lineno, n->n_end_col_offset, c->c_arena);
 }
 
 static expr_ty
@@ -2017,7 +2036,8 @@ ast_for_dictdisplay(struct compiling *c, const node *n)
     }
     keys->size = j;
     values->size = j;
-    return Dict(keys, values, LINENO(n), n->n_col_offset, c->c_arena);
+    return Dict(keys, values, LINENO(n), n->n_col_offset,
+                n->n_end_lineno, n->n_end_col_offset, c->c_arena);
 }
 
 static expr_ty
@@ -2060,7 +2080,8 @@ ast_for_setdisplay(struct compiling *c, const node *n)
             return NULL;
         asdl_seq_SET(elts, i / 2, expression);
     }
-    return Set(elts, LINENO(n), n->n_col_offset, c->c_arena);
+    return Set(elts, LINENO(n), n->n_col_offset,
+               n->n_end_lineno, n->n_end_col_offset, c->c_arena);
 }
 
 static expr_ty
@@ -2079,17 +2100,21 @@ ast_for_atom(struct compiling *c, const node *n)
         size_t len = strlen(s);
         if (len >= 4 && len <= 5) {
             if (!strcmp(s, "None"))
-                return Constant(Py_None, LINENO(n), n->n_col_offset, c->c_arena);
+                return Constant(Py_None, LINENO(n), n->n_col_offset,
+                                n->n_end_lineno, n->n_end_col_offset, c->c_arena);
             if (!strcmp(s, "True"))
-                return Constant(Py_True, LINENO(n), n->n_col_offset, c->c_arena);
+                return Constant(Py_True, LINENO(n), n->n_col_offset,
+                                n->n_end_lineno, n->n_end_col_offset, c->c_arena);
             if (!strcmp(s, "False"))
-                return Constant(Py_False, LINENO(n), n->n_col_offset, c->c_arena);
+                return Constant(Py_False, LINENO(n), n->n_col_offset,
+                                n->n_end_lineno, n->n_end_col_offset, c->c_arena);
         }
         name = new_identifier(s, c);
         if (!name)
             return NULL;
         /* All names start in Load context, but may later be changed. */
-        return Name(name, Load, LINENO(n), n->n_col_offset, c->c_arena);
+        return Name(name, Load, LINENO(n), n->n_col_offset,
+                    n->n_end_lineno, n->n_end_col_offset, c->c_arena);
     }
     case STRING: {
         expr_ty str = parsestrplus(c, n);
@@ -2128,15 +2153,18 @@ ast_for_atom(struct compiling *c, const node *n)
             Py_DECREF(pynum);
             return NULL;
         }
-        return Constant(pynum, LINENO(n), n->n_col_offset, c->c_arena);
+        return Constant(pynum, LINENO(n), n->n_col_offset,
+                        n->n_end_lineno, n->n_end_col_offset, c->c_arena);
     }
     case ELLIPSIS: /* Ellipsis */
-        return Constant(Py_Ellipsis, LINENO(n), n->n_col_offset, c->c_arena);
+        return Constant(Py_Ellipsis, LINENO(n), n->n_col_offset,
+                        n->n_end_lineno, n->n_end_col_offset, c->c_arena);
     case LPAR: /* some parenthesized expressions */
         ch = CHILD(n, 1);
 
         if (TYPE(ch) == RPAR)
-            return Tuple(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena);
+            return Tuple(NULL, Load, LINENO(n), n->n_col_offset,
+                         n->n_end_lineno, n->n_end_col_offset, c->c_arena);
 
         if (TYPE(ch) == yield_expr)
             return ast_for_expr(c, ch);
@@ -2156,7 +2184,8 @@ ast_for_atom(struct compiling *c, const node *n)
         ch = CHILD(n, 1);
 
         if (TYPE(ch) == RSQB)
-            return List(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena);
+            return List(NULL, Load, LINENO(n), n->n_col_offset,
+                        n->n_end_lineno, n->n_end_col_offset, c->c_arena);
 
         REQ(ch, testlist_comp);
         if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) {
@@ -2164,7 +2193,8 @@ ast_for_atom(struct compiling *c, const node *n)
             if (!elts)
                 return NULL;
 
-            return List(elts, Load, LINENO(n), n->n_col_offset, c->c_arena);
+            return List(elts, Load, LINENO(n), n->n_col_offset,
+                        n->n_end_lineno, n->n_end_col_offset, c->c_arena);
         }
         else {
             return copy_location(ast_for_listcomp(c, ch), n);
@@ -2178,7 +2208,8 @@ ast_for_atom(struct compiling *c, const node *n)
         ch = CHILD(n, 1);
         if (TYPE(ch) == RBRACE) {
             /* It's an empty dict. */
-            return Dict(NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
+            return Dict(NULL, NULL, LINENO(n), n->n_col_offset,
+                        n->n_end_lineno, n->n_end_col_offset, c->c_arena);
         }
         else {
             int is_dict = (TYPE(CHILD(ch, 0)) == DOUBLESTAR);
@@ -2306,6 +2337,7 @@ ast_for_binop(struct compiling *c, const node *n)
         return NULL;
 
     result = BinOp(expr1, newoperator, expr2, LINENO(n), n->n_col_offset,
+                   CHILD(n, 2)->n_end_lineno, CHILD(n, 2)->n_end_col_offset,
                    c->c_arena);
     if (!result)
         return NULL;
@@ -2325,6 +2357,8 @@ ast_for_binop(struct compiling *c, const node *n)
 
         tmp_result = BinOp(result, newoperator, tmp,
                            LINENO(next_oper), next_oper->n_col_offset,
+                           CHILD(n, i * 2 + 2)->n_end_lineno,
+                           CHILD(n, i * 2 + 2)->n_end_col_offset,
                            c->c_arena);
         if (!tmp_result)
             return NULL;
@@ -2340,20 +2374,22 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
        subscriptlist: subscript (',' subscript)* [',']
        subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop]
      */
+    const node *n_copy = n;
     REQ(n, trailer);
     if (TYPE(CHILD(n, 0)) == LPAR) {
         if (NCH(n) == 2)
-            return Call(left_expr, NULL, NULL, LINENO(n),
-                        n->n_col_offset, c->c_arena);
+            return Call(left_expr, NULL, NULL, LINENO(n), n->n_col_offset,
+                        n->n_end_lineno, n->n_end_col_offset, c->c_arena);
         else
-            return ast_for_call(c, CHILD(n, 1), left_expr, CHILD(n, 0));
+            return ast_for_call(c, CHILD(n, 1), left_expr, CHILD(n, 0), CHILD(n, 2));
     }
     else if (TYPE(CHILD(n, 0)) == DOT) {
         PyObject *attr_id = NEW_IDENTIFIER(CHILD(n, 1));
         if (!attr_id)
             return NULL;
         return Attribute(left_expr, attr_id, Load,
-                         LINENO(n), n->n_col_offset, c->c_arena);
+                         LINENO(n), n->n_col_offset,
+                         n->n_end_lineno, n->n_end_col_offset, c->c_arena);
     }
     else {
         REQ(CHILD(n, 0), LSQB);
@@ -2364,6 +2400,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
             if (!slc)
                 return NULL;
             return Subscript(left_expr, slc, Load, LINENO(n), n->n_col_offset,
+                             n_copy->n_end_lineno, n_copy->n_end_col_offset,
                              c->c_arena);
         }
         else {
@@ -2389,7 +2426,8 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
             }
             if (!simple) {
                 return Subscript(left_expr, ExtSlice(slices, c->c_arena),
-                                 Load, LINENO(n), n->n_col_offset, c->c_arena);
+                                 Load, LINENO(n), n->n_col_offset,
+                                 n_copy->n_end_lineno, n_copy->n_end_col_offset, c->c_arena);
             }
             /* extract Index values and put them in a Tuple */
             elts = _Py_asdl_seq_new(asdl_seq_LEN(slices), c->c_arena);
@@ -2400,11 +2438,13 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
                 assert(slc->kind == Index_kind  && slc->v.Index.value);
                 asdl_seq_SET(elts, j, slc->v.Index.value);
             }
-            e = Tuple(elts, Load, LINENO(n), n->n_col_offset, c->c_arena);
+            e = Tuple(elts, Load, LINENO(n), n->n_col_offset,
+                      n->n_end_lineno, n->n_end_col_offset, c->c_arena);
             if (!e)
                 return NULL;
             return Subscript(left_expr, Index(e, c->c_arena),
-                             Load, LINENO(n), n->n_col_offset, c->c_arena);
+                             Load, LINENO(n), n->n_col_offset,
+                             n_copy->n_end_lineno, n_copy->n_end_col_offset, c->c_arena);
         }
     }
 }
@@ -2421,13 +2461,16 @@ ast_for_factor(struct compiling *c, const node *n)
     switch (TYPE(CHILD(n, 0))) {
         case PLUS:
             return UnaryOp(UAdd, expression, LINENO(n), n->n_col_offset,
+                           n->n_end_lineno, n->n_end_col_offset,
                            c->c_arena);
         case MINUS:
             return UnaryOp(USub, expression, LINENO(n), n->n_col_offset,
+                           n->n_end_lineno, n->n_end_col_offset,
                            c->c_arena);
         case TILDE:
-            return UnaryOp(Invert, expression, LINENO(n),
-                           n->n_col_offset, c->c_arena);
+            return UnaryOp(Invert, expression, LINENO(n), n->n_col_offset,
+                           n->n_end_lineno, n->n_end_col_offset,
+                           c->c_arena);
     }
     PyErr_Format(PyExc_SystemError, "unhandled factor: %d",
                  TYPE(CHILD(n, 0)));
@@ -2454,7 +2497,8 @@ ast_for_atom_expr(struct compiling *c, const node *n)
     if (nch == 1)
         return e;
     if (start && nch == 2) {
-        return Await(e, LINENO(n), n->n_col_offset, c->c_arena);
+        return Await(e, LINENO(n), n->n_col_offset,
+                     n->n_end_lineno, n->n_end_col_offset, c->c_arena);
     }
 
     for (i = start + 1; i < nch; i++) {
@@ -2471,7 +2515,8 @@ ast_for_atom_expr(struct compiling *c, const node *n)
 
     if (start) {
         /* there was an 'await' */
-        return Await(e, LINENO(n), n->n_col_offset, c->c_arena);
+        return Await(e, LINENO(n), n->n_col_offset,
+                     n->n_end_lineno, n->n_end_col_offset, c->c_arena);
     }
     else {
         return e;
@@ -2494,7 +2539,8 @@ ast_for_power(struct compiling *c, const node *n)
         expr_ty f = ast_for_expr(c, CHILD(n, NCH(n) - 1));
         if (!f)
             return NULL;
-        e = BinOp(e, Pow, f, LINENO(n), n->n_col_offset, c->c_arena);
+        e = BinOp(e, Pow, f, LINENO(n), n->n_col_offset,
+                  n->n_end_lineno, n->n_end_col_offset, c->c_arena);
     }
     return e;
 }
@@ -2510,7 +2556,8 @@ ast_for_starred(struct compiling *c, const node *n)
         return NULL;
 
     /* The Load context is changed later. */
-    return Starred(tmp, Load, LINENO(n), n->n_col_offset, c->c_arena);
+    return Starred(tmp, Load, LINENO(n), n->n_col_offset,
+                   n->n_end_lineno, n->n_end_col_offset, c->c_arena);
 }
 
 
@@ -2569,9 +2616,11 @@ ast_for_expr(struct compiling *c, const node *n)
             }
             if (!strcmp(STR(CHILD(n, 1)), "and"))
                 return BoolOp(And, seq, LINENO(n), n->n_col_offset,
+                              n->n_end_lineno, n->n_end_col_offset,
                               c->c_arena);
             assert(!strcmp(STR(CHILD(n, 1)), "or"));
-            return BoolOp(Or, seq, LINENO(n), n->n_col_offset, c->c_arena);
+            return BoolOp(Or, seq, LINENO(n), n->n_col_offset,
+                          n->n_end_lineno, n->n_end_col_offset, c->c_arena);
         case not_test:
             if (NCH(n) == 1) {
                 n = CHILD(n, 0);
@@ -2583,6 +2632,7 @@ ast_for_expr(struct compiling *c, const node *n)
                     return NULL;
 
                 return UnaryOp(Not, expression, LINENO(n), n->n_col_offset,
+                               n->n_end_lineno, n->n_end_col_offset,
                                c->c_arena);
             }
         case comparison:
@@ -2622,8 +2672,8 @@ ast_for_expr(struct compiling *c, const node *n)
                     return NULL;
                 }
 
-                return Compare(expression, ops, cmps, LINENO(n),
-                               n->n_col_offset, c->c_arena);
+                return Compare(expression, ops, cmps, LINENO(n), n->n_col_offset,
+                               n->n_end_lineno, n->n_end_col_offset, c->c_arena);
             }
             break;
 
@@ -2663,8 +2713,10 @@ ast_for_expr(struct compiling *c, const node *n)
                     return NULL;
             }
             if (is_from)
-                return YieldFrom(exp, LINENO(n), n->n_col_offset, c->c_arena);
-            return Yield(exp, LINENO(n), n->n_col_offset, c->c_arena);
+                return YieldFrom(exp, LINENO(n), n->n_col_offset,
+                                 n->n_end_lineno, n->n_end_col_offset, c->c_arena);
+            return Yield(exp, LINENO(n), n->n_col_offset,
+                         n->n_end_lineno, n->n_end_col_offset, c->c_arena);
         }
         case factor:
             if (NCH(n) == 1) {
@@ -2684,7 +2736,7 @@ ast_for_expr(struct compiling *c, const node *n)
 
 static expr_ty
 ast_for_call(struct compiling *c, const node *n, expr_ty func,
-             const node *maybegenbeg)
+             const node *maybegenbeg, const node *closepar)
 {
     /*
       arglist: argument (',' argument)*  [',']
@@ -2773,6 +2825,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func,
                     return NULL;
                 starred = Starred(e, Load, LINENO(chch),
                         chch->n_col_offset,
+                        chch->n_end_lineno, chch->n_end_col_offset,
                         c->c_arena);
                 if (!starred)
                     return NULL;
@@ -2864,7 +2917,8 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func,
         }
     }
 
-    return Call(func, args, keywords, func->lineno, func->col_offset, c->c_arena);
+    return Call(func, args, keywords, func->lineno, func->col_offset,
+                closepar->n_end_lineno, closepar->n_end_col_offset, c->c_arena);
 }
 
 static expr_ty
@@ -2887,7 +2941,8 @@ ast_for_testlist(struct compiling *c, const node* n)
         asdl_seq *tmp = seq_for_testlist(c, n);
         if (!tmp)
             return NULL;
-        return Tuple(tmp, Load, LINENO(n), n->n_col_offset, c->c_arena);
+        return Tuple(tmp, Load, LINENO(n), n->n_col_offset,
+                     n->n_end_lineno, n->n_end_col_offset, c->c_arena);
     }
 }
 
@@ -2909,7 +2964,8 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
         if (!e)
             return NULL;
 
-        return Expr(e, LINENO(n), n->n_col_offset, c->c_arena);
+        return Expr(e, LINENO(n), n->n_col_offset,
+                    n->n_end_lineno, n->n_end_col_offset, c->c_arena);
     }
     else if (TYPE(CHILD(n, 1)) == augassign) {
         expr_ty expr1, expr2;
@@ -2947,7 +3003,8 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
         if (!newoperator)
             return NULL;
 
-        return AugAssign(expr1, newoperator, expr2, LINENO(n), n->n_col_offset, c->c_arena);
+        return AugAssign(expr1, newoperator, expr2, LINENO(n), n->n_col_offset,
+                         n->n_end_lineno, n->n_end_col_offset, c->c_arena);
     }
     else if (TYPE(CHILD(n, 1)) == annassign) {
         expr_ty expr1, expr2, expr3;
@@ -3007,7 +3064,8 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
         }
         if (NCH(ann) == 2) {
             return AnnAssign(expr1, expr2, NULL, simple,
-                             LINENO(n), n->n_col_offset, c->c_arena);
+                             LINENO(n), n->n_col_offset,
+                             n->n_end_lineno, n->n_end_col_offset, c->c_arena);
         }
         else {
             ch = CHILD(ann, 3);
@@ -3016,7 +3074,8 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
                 return NULL;
             }
             return AnnAssign(expr1, expr2, expr3, simple,
-                             LINENO(n), n->n_col_offset, c->c_arena);
+                             LINENO(n), n->n_col_offset,
+                             n->n_end_lineno, n->n_end_col_offset, c->c_arena);
         }
     }
     else {
@@ -3054,7 +3113,8 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
             expression = ast_for_expr(c, value);
         if (!expression)
             return NULL;
-        return Assign(targets, expression, LINENO(n), n->n_col_offset, c->c_arena);
+        return Assign(targets, expression, LINENO(n), n->n_col_offset,
+                      n->n_end_lineno, n->n_end_col_offset, c->c_arena);
     }
 }
 
@@ -3093,7 +3153,8 @@ ast_for_del_stmt(struct compiling *c, const node *n)
     expr_list = ast_for_exprlist(c, CHILD(n, 1), Del);
     if (!expr_list)
         return NULL;
-    return Delete(expr_list, LINENO(n), n->n_col_offset, c->c_arena);
+    return Delete(expr_list, LINENO(n), n->n_col_offset,
+                  n->n_end_lineno, n->n_end_col_offset, c->c_arena);
 }
 
 static stmt_ty
@@ -3115,27 +3176,33 @@ ast_for_flow_stmt(struct compiling *c, const node *n)
     ch = CHILD(n, 0);
     switch (TYPE(ch)) {
         case break_stmt:
-            return Break(LINENO(n), n->n_col_offset, c->c_arena);
+            return Break(LINENO(n), n->n_col_offset,
+                         n->n_end_lineno, n->n_end_col_offset, c->c_arena);
         case continue_stmt:
-            return Continue(LINENO(n), n->n_col_offset, c->c_arena);
+            return Continue(LINENO(n), n->n_col_offset,
+                            n->n_end_lineno, n->n_end_col_offset, c->c_arena);
         case yield_stmt: { /* will reduce to yield_expr */
             expr_ty exp = ast_for_expr(c, CHILD(ch, 0));
             if (!exp)
                 return NULL;
-            return Expr(exp, LINENO(n), n->n_col_offset, c->c_arena);
+            return Expr(exp, LINENO(n), n->n_col_offset,
+                        n->n_end_lineno, n->n_end_col_offset, c->c_arena);
         }
         case return_stmt:
             if (NCH(ch) == 1)
-                return Return(NULL, LINENO(n), n->n_col_offset, c->c_arena);
+                return Return(NULL, LINENO(n), n->n_col_offset,
+                              n->n_end_lineno, n->n_end_col_offset, c->c_arena);
             else {
                 expr_ty expression = ast_for_testlist(c, CHILD(ch, 1));
                 if (!expression)
                     return NULL;
-                return Return(expression, LINENO(n), n->n_col_offset, c->c_arena);
+                return Return(expression, LINENO(n), n->n_col_offset,
+                              n->n_end_lineno, n->n_end_col_offset, c->c_arena);
             }
         case raise_stmt:
             if (NCH(ch) == 1)
-                return Raise(NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
+                return Raise(NULL, NULL, LINENO(n), n->n_col_offset,
+                             n->n_end_lineno, n->n_end_col_offset, c->c_arena);
             else if (NCH(ch) >= 2) {
                 expr_ty cause = NULL;
                 expr_ty expression = ast_for_expr(c, CHILD(ch, 1));
@@ -3146,7 +3213,8 @@ ast_for_flow_stmt(struct compiling *c, const node *n)
                     if (!cause)
                         return NULL;
                 }
-                return Raise(expression, cause, LINENO(n), n->n_col_offset, c->c_arena);
+                return Raise(expression, cause, LINENO(n), n->n_col_offset,
+                             n->n_end_lineno, n->n_end_col_offset, c->c_arena);
             }
             /* fall through */
         default:
@@ -3307,11 +3375,14 @@ ast_for_import_stmt(struct compiling *c, const node *n)
                 return NULL;
             asdl_seq_SET(aliases, i / 2, import_alias);
         }
-        return Import(aliases, lineno, col_offset, c->c_arena);
+        // Even though n is modified above, the end position is not changed
+        return Import(aliases, lineno, col_offset,
+                      n->n_end_lineno, n->n_end_col_offset, c->c_arena);
     }
     else if (TYPE(n) == import_from) {
         int n_children;
         int idx, ndots = 0;
+        const node *n_copy = n;
         alias_ty mod = NULL;
         identifier modname = NULL;
 
@@ -3382,6 +3453,7 @@ ast_for_import_stmt(struct compiling *c, const node *n)
         if (mod != NULL)
             modname = mod->name;
         return ImportFrom(modname, aliases, ndots, lineno, col_offset,
+                          n_copy->n_end_lineno, n_copy->n_end_col_offset,
                           c->c_arena);
     }
     PyErr_Format(PyExc_SystemError,
@@ -3408,7 +3480,8 @@ ast_for_global_stmt(struct compiling *c, const node *n)
             return NULL;
         asdl_seq_SET(s, i / 2, name);
     }
-    return Global(s, LINENO(n), n->n_col_offset, c->c_arena);
+    return Global(s, LINENO(n), n->n_col_offset,
+                  n->n_end_lineno, n->n_end_col_offset, c->c_arena);
 }
 
 static stmt_ty
@@ -3429,7 +3502,8 @@ ast_for_nonlocal_stmt(struct compiling *c, const node *n)
             return NULL;
         asdl_seq_SET(s, i / 2, name);
     }
-    return Nonlocal(s, LINENO(n), n->n_col_offset, c->c_arena);
+    return Nonlocal(s, LINENO(n), n->n_col_offset,
+                    n->n_end_lineno, n->n_end_col_offset, c->c_arena);
 }
 
 static stmt_ty
@@ -3441,7 +3515,8 @@ ast_for_assert_stmt(struct compiling *c, const node *n)
         expr_ty expression = ast_for_expr(c, CHILD(n, 1));
         if (!expression)
             return NULL;
-        return Assert(expression, NULL, LINENO(n), n->n_col_offset, c->c_arena);
+        return Assert(expression, NULL, LINENO(n), n->n_col_offset,
+                      n->n_end_lineno, n->n_end_col_offset, c->c_arena);
     }
     else if (NCH(n) == 4) {
         expr_ty expr1, expr2;
@@ -3453,7 +3528,8 @@ ast_for_assert_stmt(struct compiling *c, const node *n)
         if (!expr2)
             return NULL;
 
-        return Assert(expr1, expr2, LINENO(n), n->n_col_offset, c->c_arena);
+        return Assert(expr1, expr2, LINENO(n), n->n_col_offset,
+                      n->n_end_lineno, n->n_end_col_offset, c->c_arena);
     }
     PyErr_Format(PyExc_SystemError,
                  "improper number of parts to 'assert' statement: %d",
@@ -3527,6 +3603,20 @@ ast_for_suite(struct compiling *c, const node *n)
     return seq;
 }
 
+static void
+get_last_end_pos(asdl_seq *s, int *end_lineno, int *end_col_offset)
+{
+    int tot = asdl_seq_LEN(s);
+    // Suite should not be empty, but it is safe to just ignore it
+    // if it will ever occur.
+    if (!tot) {
+        return;
+    }
+    stmt_ty last = asdl_seq_GET(s, tot - 1);
+    *end_lineno = last->end_lineno;
+    *end_col_offset = last->end_col_offset;
+}
+
 static stmt_ty
 ast_for_if_stmt(struct compiling *c, const node *n)
 {
@@ -3534,6 +3624,7 @@ ast_for_if_stmt(struct compiling *c, const node *n)
        ['else' ':' suite]
     */
     char *s;
+    int end_lineno, end_col_offset;
 
     REQ(n, if_stmt);
 
@@ -3547,9 +3638,10 @@ ast_for_if_stmt(struct compiling *c, const node *n)
         suite_seq = ast_for_suite(c, CHILD(n, 3));
         if (!suite_seq)
             return NULL;
+        get_last_end_pos(suite_seq, &end_lineno, &end_col_offset);
 
         return If(expression, suite_seq, NULL, LINENO(n), n->n_col_offset,
-                  c->c_arena);
+                  end_lineno, end_col_offset, c->c_arena);
     }
 
     s = STR(CHILD(n, 4));
@@ -3570,9 +3662,10 @@ ast_for_if_stmt(struct compiling *c, const node *n)
         seq2 = ast_for_suite(c, CHILD(n, 6));
         if (!seq2)
             return NULL;
+        get_last_end_pos(seq2, &end_lineno, &end_col_offset);
 
         return If(expression, seq1, seq2, LINENO(n), n->n_col_offset,
-                  c->c_arena);
+                  end_lineno, end_col_offset, c->c_arena);
     }
     else if (s[2] == 'i') {
         int i, n_elif, has_else = 0;
@@ -3604,12 +3697,13 @@ ast_for_if_stmt(struct compiling *c, const node *n)
             suite_seq2 = ast_for_suite(c, CHILD(n, NCH(n) - 1));
             if (!suite_seq2)
                 return NULL;
+            get_last_end_pos(suite_seq2, &end_lineno, &end_col_offset);
 
             asdl_seq_SET(orelse, 0,
                          If(expression, suite_seq, suite_seq2,
                             LINENO(CHILD(n, NCH(n) - 6)),
                             CHILD(n, NCH(n) - 6)->n_col_offset,
-                            c->c_arena));
+                            end_lineno, end_col_offset, c->c_arena));
             /* the just-created orelse handled the last elif */
             n_elif--;
         }
@@ -3626,10 +3720,16 @@ ast_for_if_stmt(struct compiling *c, const node *n)
             if (!suite_seq)
                 return NULL;
 
+            if (orelse != NULL) {
+                get_last_end_pos(orelse, &end_lineno, &end_col_offset);
+            } else {
+                get_last_end_pos(suite_seq, &end_lineno, &end_col_offset);
+            }
             asdl_seq_SET(newobj, 0,
                          If(expression, suite_seq, orelse,
                             LINENO(CHILD(n, off)),
-                            CHILD(n, off)->n_col_offset, c->c_arena));
+                            CHILD(n, off)->n_col_offset,
+                            end_lineno, end_col_offset, c->c_arena));
             orelse = newobj;
         }
         expression = ast_for_expr(c, CHILD(n, 1));
@@ -3638,8 +3738,10 @@ ast_for_if_stmt(struct compiling *c, const node *n)
         suite_seq = ast_for_suite(c, CHILD(n, 3));
         if (!suite_seq)
             return NULL;
+        get_last_end_pos(orelse, &end_lineno, &end_col_offset);
         return If(expression, suite_seq, orelse,
-                  LINENO(n), n->n_col_offset, c->c_arena);
+                  LINENO(n), n->n_col_offset,
+                  end_lineno, end_col_offset, c->c_arena);
     }
 
     PyErr_Format(PyExc_SystemError,
@@ -3652,6 +3754,7 @@ ast_for_while_stmt(struct compiling *c, const node *n)
 {
     /* while_stmt: 'while' test ':' suite ['else' ':' suite] */
     REQ(n, while_stmt);
+    int end_lineno, end_col_offset;
 
     if (NCH(n) == 4) {
         expr_ty expression;
@@ -3663,7 +3766,9 @@ ast_for_while_stmt(struct compiling *c, const node *n)
         suite_seq = ast_for_suite(c, CHILD(n, 3));
         if (!suite_seq)
             return NULL;
-        return While(expression, suite_seq, NULL, LINENO(n), n->n_col_offset, c->c_arena);
+        get_last_end_pos(suite_seq, &end_lineno, &end_col_offset);
+        return While(expression, suite_seq, NULL, LINENO(n), n->n_col_offset,
+                     end_lineno, end_col_offset, c->c_arena);
     }
     else if (NCH(n) == 7) {
         expr_ty expression;
@@ -3678,8 +3783,10 @@ ast_for_while_stmt(struct compiling *c, const node *n)
         seq2 = ast_for_suite(c, CHILD(n, 6));
         if (!seq2)
             return NULL;
+        get_last_end_pos(seq2, &end_lineno, &end_col_offset);
 
-        return While(expression, seq1, seq2, LINENO(n), n->n_col_offset, c->c_arena);
+        return While(expression, seq1, seq2, LINENO(n), n->n_col_offset,
+                     end_lineno, end_col_offset, c->c_arena);
     }
 
     PyErr_Format(PyExc_SystemError,
@@ -3696,6 +3803,7 @@ ast_for_for_stmt(struct compiling *c, const node *n0, bool is_async)
     expr_ty expression;
     expr_ty target, first;
     const node *node_target;
+    int end_lineno, end_col_offset;
     /* for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] */
     REQ(n, for_stmt);
 
@@ -3715,7 +3823,9 @@ ast_for_for_stmt(struct compiling *c, const node *n0, bool is_async)
     if (NCH(node_target) == 1)
         target = first;
     else
-        target = Tuple(_target, Store, first->lineno, first->col_offset, c->c_arena);
+        target = Tuple(_target, Store, first->lineno, first->col_offset,
+                       node_target->n_end_lineno, node_target->n_end_col_offset,
+                       c->c_arena);
 
     expression = ast_for_testlist(c, CHILD(n, 3));
     if (!expression)
@@ -3724,20 +3834,26 @@ ast_for_for_stmt(struct compiling *c, const node *n0, bool is_async)
     if (!suite_seq)
         return NULL;
 
+    if (seq != NULL) {
+        get_last_end_pos(seq, &end_lineno, &end_col_offset);
+    } else {
+        get_last_end_pos(suite_seq, &end_lineno, &end_col_offset);
+    }
     if (is_async)
         return AsyncFor(target, expression, suite_seq, seq,
                         LINENO(n0), n0->n_col_offset,
-                        c->c_arena);
+                        end_lineno, end_col_offset, c->c_arena);
     else
         return For(target, expression, suite_seq, seq,
                    LINENO(n), n->n_col_offset,
-                   c->c_arena);
+                   end_lineno, end_col_offset, c->c_arena);
 }
 
 static excepthandler_ty
 ast_for_except_clause(struct compiling *c, const node *exc, node *body)
 {
     /* except_clause: 'except' [test ['as' test]] */
+    int end_lineno, end_col_offset;
     REQ(exc, except_clause);
     REQ(body, suite);
 
@@ -3745,9 +3861,11 @@ ast_for_except_clause(struct compiling *c, const node *exc, node *body)
         asdl_seq *suite_seq = ast_for_suite(c, body);
         if (!suite_seq)
             return NULL;
+        get_last_end_pos(suite_seq, &end_lineno, &end_col_offset);
 
         return ExceptHandler(NULL, NULL, suite_seq, LINENO(exc),
-                             exc->n_col_offset, c->c_arena);
+                             exc->n_col_offset,
+                             end_lineno, end_col_offset, c->c_arena);
     }
     else if (NCH(exc) == 2) {
         expr_ty expression;
@@ -3759,9 +3877,11 @@ ast_for_except_clause(struct compiling *c, const node *exc, node *body)
         suite_seq = ast_for_suite(c, body);
         if (!suite_seq)
             return NULL;
+        get_last_end_pos(suite_seq, &end_lineno, &end_col_offset);
 
         return ExceptHandler(expression, NULL, suite_seq, LINENO(exc),
-                             exc->n_col_offset, c->c_arena);
+                             exc->n_col_offset,
+                             end_lineno, end_col_offset, c->c_arena);
     }
     else if (NCH(exc) == 4) {
         asdl_seq *suite_seq;
@@ -3777,9 +3897,11 @@ ast_for_except_clause(struct compiling *c, const node *exc, node *body)
         suite_seq = ast_for_suite(c, body);
         if (!suite_seq)
             return NULL;
+        get_last_end_pos(suite_seq, &end_lineno, &end_col_offset);
 
         return ExceptHandler(expression, e, suite_seq, LINENO(exc),
-                             exc->n_col_offset, c->c_arena);
+                             exc->n_col_offset,
+                             end_lineno, end_col_offset, c->c_arena);
     }
 
     PyErr_Format(PyExc_SystemError,
@@ -3792,8 +3914,9 @@ static stmt_ty
 ast_for_try_stmt(struct compiling *c, const node *n)
 {
     const int nch = NCH(n);
-    int n_except = (nch - 3)/3;
+    int end_lineno, end_col_offset, n_except = (nch - 3)/3;
     asdl_seq *body, *handlers = NULL, *orelse = NULL, *finally = NULL;
+    excepthandler_ty last_handler;
 
     REQ(n, try_stmt);
 
@@ -3849,7 +3972,20 @@ ast_for_try_stmt(struct compiling *c, const node *n)
     }
 
     assert(finally != NULL || asdl_seq_LEN(handlers));
-    return Try(body, handlers, orelse, finally, LINENO(n), n->n_col_offset, c->c_arena);
+        if (finally != NULL) {
+        // finally is always last
+        get_last_end_pos(finally, &end_lineno, &end_col_offset);
+    } else if (orelse != NULL) {
+        // otherwise else is last
+        get_last_end_pos(orelse, &end_lineno, &end_col_offset);
+    } else {
+        // inline the get_last_end_pos logic due to layout mismatch
+        last_handler = (excepthandler_ty) asdl_seq_GET(handlers, n_except - 1);
+        end_lineno = last_handler->end_lineno;
+        end_col_offset = last_handler->end_col_offset;
+    }
+    return Try(body, handlers, orelse, finally, LINENO(n), n->n_col_offset,
+               end_lineno, end_col_offset, c->c_arena);
 }
 
 /* with_item: test ['as' expr] */
@@ -3881,7 +4017,7 @@ static stmt_ty
 ast_for_with_stmt(struct compiling *c, const node *n0, bool is_async)
 {
     const node * const n = is_async ? CHILD(n0, 1) : n0;
-    int i, n_items;
+    int i, n_items, end_lineno, end_col_offset;
     asdl_seq *items, *body;
 
     REQ(n, with_stmt);
@@ -3900,11 +4036,14 @@ ast_for_with_stmt(struct compiling *c, const node *n0, bool is_async)
     body = ast_for_suite(c, CHILD(n, NCH(n) - 1));
     if (!body)
         return NULL;
+    get_last_end_pos(body, &end_lineno, &end_col_offset);
 
     if (is_async)
-        return AsyncWith(items, body, LINENO(n0), n0->n_col_offset, c->c_arena);
+        return AsyncWith(items, body, LINENO(n0), n0->n_col_offset,
+                         end_lineno, end_col_offset, c->c_arena);
     else
-        return With(items, body, LINENO(n), n->n_col_offset, c->c_arena);
+        return With(items, body, LINENO(n), n->n_col_offset,
+                    end_lineno, end_col_offset, c->c_arena);
 }
 
 static stmt_ty
@@ -3914,6 +4053,7 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
     PyObject *classname;
     asdl_seq *s;
     expr_ty call;
+    int end_lineno, end_col_offset;
 
     REQ(n, classdef);
 
@@ -3921,26 +4061,32 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
         s = ast_for_suite(c, CHILD(n, 3));
         if (!s)
             return NULL;
+        get_last_end_pos(s, &end_lineno, &end_col_offset);
+
         classname = NEW_IDENTIFIER(CHILD(n, 1));
         if (!classname)
             return NULL;
         if (forbidden_name(c, classname, CHILD(n, 3), 0))
             return NULL;
         return ClassDef(classname, NULL, NULL, s, decorator_seq,
-                        LINENO(n), n->n_col_offset, c->c_arena);
+                        LINENO(n), n->n_col_offset,
+                        end_lineno, end_col_offset, c->c_arena);
     }
 
     if (TYPE(CHILD(n, 3)) == RPAR) { /* class NAME '(' ')' ':' suite */
         s = ast_for_suite(c, CHILD(n, 5));
         if (!s)
             return NULL;
+        get_last_end_pos(s, &end_lineno, &end_col_offset);
+
         classname = NEW_IDENTIFIER(CHILD(n, 1));
         if (!classname)
             return NULL;
         if (forbidden_name(c, classname, CHILD(n, 3), 0))
             return NULL;
         return ClassDef(classname, NULL, NULL, s, decorator_seq,
-                        LINENO(n), n->n_col_offset, c->c_arena);
+                        LINENO(n), n->n_col_offset,
+                        end_lineno, end_col_offset, c->c_arena);
     }
 
     /* class NAME '(' arglist ')' ':' suite */
@@ -3951,14 +4097,18 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
         dummy_name = NEW_IDENTIFIER(CHILD(n, 1));
         if (!dummy_name)
             return NULL;
-        dummy = Name(dummy_name, Load, LINENO(n), n->n_col_offset, c->c_arena);
-        call = ast_for_call(c, CHILD(n, 3), dummy, NULL);
+        dummy = Name(dummy_name, Load, LINENO(n), n->n_col_offset,
+                     CHILD(n, 1)->n_end_lineno, CHILD(n, 1)->n_end_col_offset,
+                     c->c_arena);
+        call = ast_for_call(c, CHILD(n, 3), dummy, NULL, CHILD(n, 4));
         if (!call)
             return NULL;
     }
     s = ast_for_suite(c, CHILD(n, 6));
     if (!s)
         return NULL;
+    get_last_end_pos(s, &end_lineno, &end_col_offset);
+
     classname = NEW_IDENTIFIER(CHILD(n, 1));
     if (!classname)
         return NULL;
@@ -3966,7 +4116,8 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
         return NULL;
 
     return ClassDef(classname, call->v.Call.args, call->v.Call.keywords, s,
-                    decorator_seq, LINENO(n), n->n_col_offset, c->c_arena);
+                    decorator_seq, LINENO(n), n->n_col_offset,
+                    end_lineno, end_col_offset, c->c_arena);
 }
 
 static stmt_ty
@@ -3991,7 +4142,8 @@ ast_for_stmt(struct compiling *c, const node *n)
             case del_stmt:
                 return ast_for_del_stmt(c, n);
             case pass_stmt:
-                return Pass(LINENO(n), n->n_col_offset, c->c_arena);
+                return Pass(LINENO(n), n->n_col_offset,
+                            n->n_end_lineno, n->n_end_col_offset, c->c_arena);
             case flow_stmt:
                 return ast_for_flow_stmt(c, n);
             case import_stmt:
@@ -4248,6 +4400,7 @@ decode_bytes_with_escapes(struct compiling *c, const node *n, const char *s,
 static void fstring_shift_node_locations(node *n, int lineno, int col_offset)
 {
     n->n_col_offset = n->n_col_offset + col_offset;
+    n->n_end_col_offset = n->n_end_col_offset + col_offset;
     for (int i = 0; i < NCH(n); ++i) {
         if (n->n_lineno && n->n_lineno < CHILD(n, i)->n_lineno) {
             /* Shifting column offsets unnecessary if there's been newlines. */
@@ -4256,6 +4409,7 @@ static void fstring_shift_node_locations(node *n, int lineno, int col_offset)
         fstring_shift_node_locations(CHILD(n, i), lineno, col_offset);
     }
     n->n_lineno = n->n_lineno + lineno;
+    n->n_end_lineno = n->n_end_lineno + lineno;
 }
 
 /* Fix locations for the given node and its children.
@@ -4672,6 +4826,7 @@ fstring_find_expr(const char **str, const char *end, int raw, int recurse_lvl,
        entire expression with the conversion and format spec. */
     *expression = FormattedValue(simple_expression, conversion,
                                  format_spec, LINENO(n), n->n_col_offset,
+                                 n->n_end_lineno, n->n_end_col_offset,
                                  c->c_arena);
     if (!*expression)
         return -1;
@@ -4918,7 +5073,8 @@ make_str_node_and_del(PyObject **str, struct compiling *c, const node* n)
         Py_DECREF(s);
         return NULL;
     }
-    return Constant(s, LINENO(n), n->n_col_offset, c->c_arena);
+    return Constant(s, LINENO(n), n->n_col_offset,
+                    n->n_end_lineno, n->n_end_col_offset, c->c_arena);
 }
 
 /* Add a non-f-string (that is, a regular literal string). str is
@@ -5073,7 +5229,8 @@ FstringParser_Finish(FstringParser *state, struct compiling *c,
     if (!seq)
         goto error;
 
-    return JoinedStr(seq, LINENO(n), n->n_col_offset, c->c_arena);
+    return JoinedStr(seq, LINENO(n), n->n_col_offset,
+                     n->n_end_lineno, n->n_end_col_offset, c->c_arena);
 
 error:
     FstringParser_Dealloc(state);
@@ -5283,7 +5440,8 @@ parsestrplus(struct compiling *c, const node *n)
         /* Just return the bytes object and we're done. */
         if (PyArena_AddPyObject(c->c_arena, bytes_str) < 0)
             goto error;
-        return Constant(bytes_str, LINENO(n), n->n_col_offset, c->c_arena);
+        return Constant(bytes_str, LINENO(n), n->n_col_offset,
+                        n->n_end_lineno, n->n_end_col_offset, c->c_arena);
     }
 
     /* We're not a bytes string, bytes_str should never have been set. */

Python/ast_opt.c

@@ -439,7 +439,8 @@ astfold_body(asdl_seq *stmts, PyArena *ctx_, int optimize_)
             return 0;
         }
         asdl_seq_SET(values, 0, st->v.Expr.value);
-        expr_ty expr = JoinedStr(values, st->lineno, st->col_offset, ctx_);
+        expr_ty expr = JoinedStr(values, st->lineno, st->col_offset,
+                                 st->end_lineno, st->end_col_offset, ctx_);
         if (!expr) {
             return 0;
         }

Python/compile.c

@@ -4757,7 +4757,8 @@ compiler_augassign(struct compiler *c, stmt_ty s)
     switch (e->kind) {
     case Attribute_kind:
         auge = Attribute(e->v.Attribute.value, e->v.Attribute.attr,
-                         AugLoad, e->lineno, e->col_offset, c->c_arena);
+                         AugLoad, e->lineno, e->col_offset,
+                         e->end_lineno, e->end_col_offset, c->c_arena);
         if (auge == NULL)
             return 0;
         VISIT(c, expr, auge);
@@ -4768,7 +4769,8 @@ compiler_augassign(struct compiler *c, stmt_ty s)
         break;
     case Subscript_kind:
         auge = Subscript(e->v.Subscript.value, e->v.Subscript.slice,
-                         AugLoad, e->lineno, e->col_offset, c->c_arena);
+                         AugLoad, e->lineno, e->col_offset,
+                         e->end_lineno, e->end_col_offset, c->c_arena);
         if (auge == NULL)
             return 0;
         VISIT(c, expr, auge);