bpo-43244: Remove symtable.h header file (GH-24910)

Rename Include/symtable.h to to Include/internal/pycore_symtable.h,
don't export symbols anymore (replace PyAPI_FUNC and PyAPI_DATA with
extern) and rename functions:

* PyST_GetScope() to _PyST_GetScope()
* PySymtable_BuildObject() to _PySymtable_Build()
* PySymtable_Free() to _PySymtable_Free()

Remove PySymtable_Build(), Py_SymtableString() and
Py_SymtableStringObject() functions.

The Py_SymtableString() function was part the stable ABI by mistake
but it could not be used, since the symtable.h header file was
excluded from the limited C API.

The Python symtable module remains available and is unchanged.

Doc/data/stable_abi.dat

@@ -776,7 +776,6 @@ Py_SetPath
 Py_SetProgramName
 Py_SetPythonHome
 Py_SetRecursionLimit
-Py_SymtableString
 Py_UTF8Mode
 Py_VaBuildValue
 Py_XNewRef

Doc/whatsnew/3.10.rst

@@ -1364,3 +1364,19 @@ Removed
   AST object (``mod_ty`` type) with the public C API. The function was already
   excluded from the limited C API (:pep:`384`).
   (Contributed by Victor Stinner in :issue:`43244`.)
+
+* Remove the ``symtable.h`` header file and the undocumented functions:
+
+  * ``PyST_GetScope()``
+  * ``PySymtable_Build()``
+  * ``PySymtable_BuildObject()``
+  * ``PySymtable_Free()``
+  * ``Py_SymtableString()``
+  * ``Py_SymtableStringObject()``
+
+  The ``Py_SymtableString()`` function was part the stable ABI by mistake but
+  it could not be used, because the ``symtable.h`` header file was excluded
+  from the limited C API.
+
+  The Python :mod:`symtable` module remains available and is unchanged.
+  (Contributed by Victor Stinner in :issue:`43244`.)

Include/cpython/pythonrun.h

@@ -77,17 +77,6 @@ PyAPI_FUNC(const char *) _Py_SourceAsString(
     PyCompilerFlags *cf,
     PyObject **cmd_copy);
 
-PyAPI_FUNC(struct symtable *) Py_SymtableStringObject(
-    const char *str,
-    PyObject *filename,
-    int start);
-
-PyAPI_FUNC(struct symtable *) _Py_SymtableStringObjectFlags(
-    const char *str,
-    PyObject *filename,
-    int start,
-    PyCompilerFlags *flags);
-
 
 /* A function flavor is also exported by libpython. It is required when
     libpython is accessed directly rather than using header files which defines

Include/internal/pycore_symtable.h

@@ -1,15 +1,14 @@
-#ifndef Py_LIMITED_API
-#ifndef Py_SYMTABLE_H
-#define Py_SYMTABLE_H
+#ifndef Py_INTERNAL_SYMTABLE_H
+#define Py_INTERNAL_SYMTABLE_H
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#include "Python-ast.h"   /* mod_ty */
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
 
-/* XXX(ncoghlan): This is a weird mix of public names and interpreter internal
- *                names.
- */
+#include "Python-ast.h"   /* mod_ty */
 
 typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock }
     _Py_block_ty;
@@ -68,23 +67,19 @@ typedef struct _symtable_entry {
     struct symtable *ste_table;
 } PySTEntryObject;
 
-PyAPI_DATA(PyTypeObject) PySTEntry_Type;
+extern PyTypeObject PySTEntry_Type;
 
 #define PySTEntry_Check(op) Py_IS_TYPE(op, &PySTEntry_Type)
 
-PyAPI_FUNC(int) PyST_GetScope(PySTEntryObject *, PyObject *);
+extern int _PyST_GetScope(PySTEntryObject *, PyObject *);
 
-PyAPI_FUNC(struct symtable *) PySymtable_Build(
-    mod_ty mod,
-    const char *filename,       /* decoded from the filesystem encoding */
-    PyFutureFeatures *future);
-PyAPI_FUNC(struct symtable *) PySymtable_BuildObject(
+extern struct symtable* _PySymtable_Build(
     mod_ty mod,
     PyObject *filename,
     PyFutureFeatures *future);
 PyAPI_FUNC(PySTEntryObject *) PySymtable_Lookup(struct symtable *, void *);
 
-PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
+extern void _PySymtable_Free(struct symtable *);
 
 /* Flags for def-use information */
 
@@ -117,8 +112,14 @@ PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
 #define GENERATOR 1
 #define GENERATOR_EXPRESSION 2
 
+// Used by symtablemodule.c
+extern struct symtable* _Py_SymtableStringObjectFlags(
+    const char *str,
+    PyObject *filename,
+    int start,
+    PyCompilerFlags *flags);
+
 #ifdef __cplusplus
 }
 #endif
-#endif /* !Py_SYMTABLE_H */
-#endif /* !Py_LIMITED_API */
+#endif /* !Py_INTERNAL_SYMTABLE_H */

Include/pythonrun.h

@@ -9,11 +9,6 @@ extern "C" {
 
 PyAPI_FUNC(PyObject *) Py_CompileString(const char *, const char *, int);
 
-PyAPI_FUNC(struct symtable *) Py_SymtableString(
-    const char *str,
-    const char *filename,       /* decoded from the filesystem encoding */
-    int start);
-
 PyAPI_FUNC(void) PyErr_Print(void);
 PyAPI_FUNC(void) PyErr_PrintEx(int);
 PyAPI_FUNC(void) PyErr_Display(PyObject *, PyObject *, PyObject *);

Makefile.pre.in

@@ -1087,7 +1087,6 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/sliceobject.h \
 		$(srcdir)/Include/structmember.h \
 		$(srcdir)/Include/structseq.h \
-		$(srcdir)/Include/symtable.h \
 		$(srcdir)/Include/sysmodule.h \
 		$(srcdir)/Include/token.h \
 		$(srcdir)/Include/traceback.h \
@@ -1167,6 +1166,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/internal/pycore_pymem.h \
 		$(srcdir)/Include/internal/pycore_pystate.h \
 		$(srcdir)/Include/internal/pycore_runtime.h \
+		$(srcdir)/Include/internal/pycore_symtable.h \
 		$(srcdir)/Include/internal/pycore_sysmodule.h \
 		$(srcdir)/Include/internal/pycore_traceback.h \
 		$(srcdir)/Include/internal/pycore_tuple.h \

Misc/NEWS.d/next/C API/2021-03-17-23-53-14.bpo-43244.kfPqA_.rst

@@ -0,0 +1,16 @@
+Remove the ``symtable.h`` header file and the undocumented functions:
+
+* ``PyST_GetScope()``
+* ``PySymtable_Build()``
+* ``PySymtable_BuildObject()``
+* ``PySymtable_Free()``
+* ``Py_SymtableString()``
+* ``Py_SymtableStringObject()``
+
+The ``Py_SymtableString()`` function was part the stable ABI by mistake but it
+could not be used, because the ``symtable.h`` header file was excluded from the
+limited C API.
+
+The Python :mod:`symtable` module remains available and is unchanged.
+
+Patch by Victor Stinner.

Modules/symtablemodule.c

@@ -1,8 +1,7 @@
 #include "Python.h"
 
-#include "code.h"
 #include "Python-ast.h"
-#include "symtable.h"
+#include "pycore_symtable.h"      // struct symtable
 
 #include "clinic/symtablemodule.c.h"
 /*[clinic input]
@@ -62,7 +61,7 @@ _symtable_symtable_impl(PyObject *module, PyObject *source,
     t = (PyObject *)st->st_top;
     Py_INCREF(t);
     PyMem_Free((void *)st->st_future);
-    PySymtable_Free(st);
+    _PySymtable_Free(st);
     return t;
 }
 

PC/python3dll.c

@@ -80,7 +80,6 @@ EXPORT_FUNC(Py_SetPath)
 EXPORT_FUNC(Py_SetProgramName)
 EXPORT_FUNC(Py_SetPythonHome)
 EXPORT_FUNC(Py_SetRecursionLimit)
-EXPORT_FUNC(Py_SymtableString)
 EXPORT_FUNC(Py_VaBuildValue)
 EXPORT_FUNC(Py_XNewRef)
 EXPORT_FUNC(PyArg_Parse)

PCbuild/pythoncore.vcxproj

@@ -208,6 +208,7 @@
     <ClInclude Include="..\Include\internal\pycore_pystate.h" />
     <ClInclude Include="..\Include\internal\pycore_runtime.h" />
     <ClInclude Include="..\Include\internal\pycore_sysmodule.h" />
+    <ClInclude Include="..\Include\internal\pycore_symtable.h" />
     <ClInclude Include="..\Include\internal\pycore_traceback.h" />
     <ClInclude Include="..\Include\internal\pycore_tuple.h" />
     <ClInclude Include="..\Include\internal\pycore_ucnhash.h" />

PCbuild/pythoncore.vcxproj.filters

@@ -585,6 +585,9 @@
     <ClInclude Include="..\Include\internal\pycore_sysmodule.h">
       <Filter>Include\internal</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\internal\pycore_symtable.h">
+      <Filter>Include\internal</Filter>
+    </ClInclude>
     <ClInclude Include="..\Include\internal\pycore_traceback.h">
       <Filter>Include\internal</Filter>
     </ClInclude>

Python/compile.c

@@ -25,8 +25,8 @@
 #include "pycore_ast.h"           // _PyAST_GetDocString()
 #include "pycore_pymem.h"         // _PyMem_IsPtrFreed()
 #include "pycore_long.h"          // _PyLong_GetZero()
+#include "pycore_symtable.h"      // PySTEntryObject
 
-#include "symtable.h"             // struct symtable
 #define NEED_OPCODE_JUMP_TABLES
 #include "opcode.h"               // EXTENDED_ARG
 #include "wordcode_helpers.h"     // instrsize()
@@ -394,7 +394,7 @@ PyAST_CompileObject(mod_ty mod, PyObject *filename, PyCompilerFlags *flags,
         goto finally;
     }
 
-    c.c_st = PySymtable_BuildObject(mod, filename, c.c_future);
+    c.c_st = _PySymtable_Build(mod, filename, c.c_future);
     if (c.c_st == NULL) {
         if (!PyErr_Occurred())
             PyErr_SetString(PyExc_SystemError, "no symtable");
@@ -428,7 +428,7 @@ static void
 compiler_free(struct compiler *c)
 {
     if (c->c_st)
-        PySymtable_Free(c->c_st);
+        _PySymtable_Free(c->c_st);
     if (c->c_future)
         PyObject_Free(c->c_future);
     Py_XDECREF(c->c_filename);
@@ -729,7 +729,7 @@ compiler_set_qualname(struct compiler *c)
             mangled = _Py_Mangle(parent->u_private, u->u_name);
             if (!mangled)
                 return 0;
-            scope = PyST_GetScope(parent->u_ste, mangled);
+            scope = _PyST_GetScope(parent->u_ste, mangled);
             Py_DECREF(mangled);
             assert(scope != GLOBAL_IMPLICIT);
             if (scope == GLOBAL_EXPLICIT)
@@ -1920,10 +1920,10 @@ get_ref_type(struct compiler *c, PyObject *name)
     if (c->u->u_scope_type == COMPILER_SCOPE_CLASS &&
         _PyUnicode_EqualToASCIIString(name, "__class__"))
         return CELL;
-    scope = PyST_GetScope(c->u->u_ste, name);
+    scope = _PyST_GetScope(c->u->u_ste, name);
     if (scope == 0) {
         PyErr_Format(PyExc_SystemError,
-                     "PyST_GetScope(name=%R) failed: "
+                     "_PyST_GetScope(name=%R) failed: "
                      "unknown scope in unit %S (%R); "
                      "symbols: %R; locals: %R; globals: %R",
                      name,
@@ -3608,7 +3608,7 @@ compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx)
 
     op = 0;
     optype = OP_NAME;
-    scope = PyST_GetScope(c->u->u_ste, mangled);
+    scope = _PyST_GetScope(c->u->u_ste, mangled);
     switch (scope) {
     case FREE:
         dict = c->u->u_freevars;

Python/pythonrun.c

@@ -23,7 +23,6 @@
 #include "token.h"                // INDENT
 #include "errcode.h"              // E_EOF
 #include "code.h"                 // PyCodeObject
-#include "symtable.h"             // PySymtable_BuildObject()
 #include "marshal.h"              // PyMarshal_ReadLongFromFile()
 
 #ifdef MS_WINDOWS
@@ -1369,48 +1368,6 @@ _Py_SourceAsString(PyObject *cmd, const char *funcname, const char *what, PyComp
     return str;
 }
 
-struct symtable *
-Py_SymtableStringObject(const char *str, PyObject *filename, int start)
-{
-    PyCompilerFlags flags = _PyCompilerFlags_INIT;
-    return _Py_SymtableStringObjectFlags(str, filename, start, &flags);
-}
-
-struct symtable *
-_Py_SymtableStringObjectFlags(const char *str, PyObject *filename, int start, PyCompilerFlags *flags)
-{
-    struct symtable *st;
-    mod_ty mod;
-    PyArena *arena;
-
-    arena = PyArena_New();
-    if (arena == NULL)
-        return NULL;
-
-    mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena);
-    if (mod == NULL) {
-        PyArena_Free(arena);
-        return NULL;
-    }
-    st = PySymtable_BuildObject(mod, filename, 0);
-    PyArena_Free(arena);
-    return st;
-}
-
-struct symtable *
-Py_SymtableString(const char *str, const char *filename_str, int start)
-{
-    PyObject *filename;
-    struct symtable *st;
-
-    filename = PyUnicode_DecodeFSDefault(filename_str);
-    if (filename == NULL)
-        return NULL;
-    st = Py_SymtableStringObject(str, filename, start);
-    Py_DECREF(filename);
-    return st;
-}
-
 #if defined(USE_STACKCHECK)
 #if defined(WIN32) && defined(_MSC_VER)
 

Python/symtable.c

@@ -1,6 +1,6 @@
 #include "Python.h"
-#include "pycore_pystate.h"   // _PyThreadState_GET()
-#include "symtable.h"
+#include "pycore_pystate.h"       // _PyThreadState_GET()
+#include "pycore_symtable.h"      // PySTEntryObject
 #undef Yield   /* undefine macro conflicting with <winbase.h> */
 #include "structmember.h"         // PyMemberDef
 
@@ -243,7 +243,7 @@ symtable_new(void)
     st->in_pattern = 0;
     return st;
  fail:
-    PySymtable_Free(st);
+    _PySymtable_Free(st);
     return NULL;
 }
 
@@ -260,7 +260,7 @@ symtable_new(void)
 #define COMPILER_STACK_FRAME_SCALE 3
 
 struct symtable *
-PySymtable_BuildObject(mod_ty mod, PyObject *filename, PyFutureFeatures *future)
+_PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future)
 {
     struct symtable *st = symtable_new();
     asdl_stmt_seq *seq;
@@ -272,7 +272,7 @@ PySymtable_BuildObject(mod_ty mod, PyObject *filename, PyFutureFeatures *future)
     if (st == NULL)
         return NULL;
     if (filename == NULL) {
-        PySymtable_Free(st);
+        _PySymtable_Free(st);
         return NULL;
     }
     Py_INCREF(filename);
@@ -282,7 +282,7 @@ PySymtable_BuildObject(mod_ty mod, PyObject *filename, PyFutureFeatures *future)
     /* Setup recursion depth check counters */
     tstate = _PyThreadState_GET();
     if (!tstate) {
-        PySymtable_Free(st);
+        _PySymtable_Free(st);
         return NULL;
     }
     /* Be careful here to prevent overflow. */
@@ -295,7 +295,7 @@ PySymtable_BuildObject(mod_ty mod, PyObject *filename, PyFutureFeatures *future)
     /* Make the initial symbol information gathering pass */
     if (!GET_IDENTIFIER(top) ||
         !symtable_enter_block(st, top, ModuleBlock, (void *)mod, 0, 0)) {
-        PySymtable_Free(st);
+        _PySymtable_Free(st);
         return NULL;
     }
 
@@ -325,7 +325,7 @@ PySymtable_BuildObject(mod_ty mod, PyObject *filename, PyFutureFeatures *future)
         goto error;
     }
     if (!symtable_exit_block(st)) {
-        PySymtable_Free(st);
+        _PySymtable_Free(st);
         return NULL;
     }
     /* Check that the recursion depth counting balanced correctly */
@@ -333,35 +333,23 @@ PySymtable_BuildObject(mod_ty mod, PyObject *filename, PyFutureFeatures *future)
         PyErr_Format(PyExc_SystemError,
             "symtable analysis recursion depth mismatch (before=%d, after=%d)",
             starting_recursion_depth, st->recursion_depth);
-        PySymtable_Free(st);
+        _PySymtable_Free(st);
         return NULL;
     }
     /* Make the second symbol analysis pass */
     if (symtable_analyze(st))
         return st;
-    PySymtable_Free(st);
+    _PySymtable_Free(st);
     return NULL;
  error:
     (void) symtable_exit_block(st);
-    PySymtable_Free(st);
+    _PySymtable_Free(st);
     return NULL;
 }
 
-struct symtable *
-PySymtable_Build(mod_ty mod, const char *filename_str, PyFutureFeatures *future)
-{
-    PyObject *filename;
-    struct symtable *st;
-    filename = PyUnicode_DecodeFSDefault(filename_str);
-    if (filename == NULL)
-        return NULL;
-    st = PySymtable_BuildObject(mod, filename, future);
-    Py_DECREF(filename);
-    return st;
-}
 
 void
-PySymtable_Free(struct symtable *st)
+_PySymtable_Free(struct symtable *st)
 {
     Py_XDECREF(st->st_filename);
     Py_XDECREF(st->st_blocks);
@@ -402,7 +390,7 @@ _PyST_GetSymbol(PySTEntryObject *ste, PyObject *name)
 }
 
 int
-PyST_GetScope(PySTEntryObject *ste, PyObject *name)
+_PyST_GetScope(PySTEntryObject *ste, PyObject *name)
 {
     long symbol = _PyST_GetSymbol(ste, name);
     return (symbol >> SCOPE_OFFSET) & SCOPE_MASK;
@@ -1971,3 +1959,26 @@ symtable_visit_dictcomp(struct symtable *st, expr_ty e)
                                          e->v.DictComp.key,
                                          e->v.DictComp.value);
 }
+
+
+struct symtable *
+_Py_SymtableStringObjectFlags(const char *str, PyObject *filename,
+                              int start, PyCompilerFlags *flags)
+{
+    struct symtable *st;
+    mod_ty mod;
+    PyArena *arena;
+
+    arena = PyArena_New();
+    if (arena == NULL)
+        return NULL;
+
+    mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena);
+    if (mod == NULL) {
+        PyArena_Free(arena);
+        return NULL;
+    }
+    st = _PySymtable_Build(mod, filename, 0);
+    PyArena_Free(arena);
+    return st;
+}

Tools/scripts/stable_abi.py

@@ -24,7 +24,6 @@
     "parsetok.h",
     "pyatomic.h",
     "pytime.h",
-    "symtable.h",
     "token.h",
     "ucnhash.h",
 }