gh-101524: Only Use Public C-API in the _xxsubinterpreters Module (gh-107359)

The _xxsubinterpreters module should not rely on internal API.  Some of the functions it uses were recently moved there however.  Here we move them back (and expose them properly).

Include/cpython/interpreteridobject.h

@@ -0,0 +1,11 @@
+#ifndef Py_CPYTHON_INTERPRETERIDOBJECT_H
+#  error "this header file must not be included directly"
+#endif
+
+/* Interpreter ID Object */
+
+PyAPI_DATA(PyTypeObject) PyInterpreterID_Type;
+
+PyAPI_FUNC(PyObject *) PyInterpreterID_New(int64_t);
+PyAPI_FUNC(PyObject *) PyInterpreterState_GetIDObject(PyInterpreterState *);
+PyAPI_FUNC(PyInterpreterState *) PyInterpreterID_LookUp(PyObject *);

Include/cpython/pylifecycle.h

@@ -77,3 +77,7 @@ typedef struct {
 PyAPI_FUNC(PyStatus) Py_NewInterpreterFromConfig(
     PyThreadState **tstate_p,
     const PyInterpreterConfig *config);
+
+typedef void (*atexit_datacallbackfunc)(void *);
+PyAPI_FUNC(int) PyUnstable_AtExit(
+        PyInterpreterState *, atexit_datacallbackfunc, void *);

Include/cpython/pystate.h

@@ -8,6 +8,7 @@
 PyAPI_FUNC(int) _PyInterpreterState_RequiresIDRef(PyInterpreterState *);
 PyAPI_FUNC(void) _PyInterpreterState_RequireIDRef(PyInterpreterState *, int);
 
+PyAPI_FUNC(PyObject *) PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *);
 
 /* State unique per thread */
 

Include/internal/pycore_interp.h

@@ -238,9 +238,6 @@ extern int _PyInterpreterState_IDInitref(PyInterpreterState *);
 extern int _PyInterpreterState_IDIncref(PyInterpreterState *);
 extern void _PyInterpreterState_IDDecref(PyInterpreterState *);
 
-// Export for '_xxsubinterpreters' shared extension
-PyAPI_FUNC(PyObject*) _PyInterpreterState_GetMainModule(PyInterpreterState *);
-
 extern const PyConfig* _PyInterpreterState_GetConfig(PyInterpreterState *interp);
 
 /* Get a copy of the current interpreter configuration.

Include/internal/pycore_interp_id.h

@@ -1,28 +0,0 @@
-/* Interpreter ID Object */
-
-#ifndef Py_INTERNAL_INTERPRETERIDOBJECT_H
-#define Py_INTERNAL_INTERPRETERIDOBJECT_H
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#ifndef Py_BUILD_CORE
-#  error "this header requires Py_BUILD_CORE define"
-#endif
-
-// Export for '_xxsubinterpreters' shared extension
-PyAPI_DATA(PyTypeObject) _PyInterpreterID_Type;
-
-// Export for '_xxsubinterpreters' shared extension
-PyAPI_FUNC(PyObject*) _PyInterpreterID_New(int64_t);
-
-// Export for '_xxinterpchannels' shared extension
-PyAPI_FUNC(PyObject*) _PyInterpreterState_GetIDObject(PyInterpreterState *);
-
-// Export for '_testinternalcapi' shared extension
-PyAPI_FUNC(PyInterpreterState*) _PyInterpreterID_LookUp(PyObject *);
-
-#ifdef __cplusplus
-}
-#endif
-#endif  // !Py_INTERNAL_INTERPRETERIDOBJECT_H

Include/interpreteridobject.h

@@ -0,0 +1,17 @@
+#ifndef Py_INTERPRETERIDOBJECT_H
+#define Py_INTERPRETERIDOBJECT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_LIMITED_API
+#  define Py_CPYTHON_INTERPRETERIDOBJECT_H
+#  include "cpython/interpreteridobject.h"
+#  undef Py_CPYTHON_INTERPRETERIDOBJECT_H
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERPRETERIDOBJECT_H */

Makefile.pre.in

@@ -1631,6 +1631,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/floatobject.h \
 		$(srcdir)/Include/frameobject.h \
 		$(srcdir)/Include/import.h \
+		$(srcdir)/Include/interpreteridobject.h \
 		$(srcdir)/Include/intrcheck.h \
 		$(srcdir)/Include/iterobject.h \
 		$(srcdir)/Include/listobject.h \
@@ -1700,6 +1701,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/cpython/genobject.h \
 		$(srcdir)/Include/cpython/import.h \
 		$(srcdir)/Include/cpython/initconfig.h \
+		$(srcdir)/Include/cpython/interpreteridobject.h \
 		$(srcdir)/Include/cpython/listobject.h \
 		$(srcdir)/Include/cpython/longintrepr.h \
 		$(srcdir)/Include/cpython/longobject.h \
@@ -1771,7 +1773,6 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/internal/pycore_import.h \
 		$(srcdir)/Include/internal/pycore_initconfig.h \
 		$(srcdir)/Include/internal/pycore_interp.h \
-		$(srcdir)/Include/internal/pycore_interp_id.h \
 		$(srcdir)/Include/internal/pycore_intrinsics.h \
 		$(srcdir)/Include/internal/pycore_list.h \
 		$(srcdir)/Include/internal/pycore_long.h \

Modules/_testinternalcapi.c

@@ -13,27 +13,27 @@
 #include "pycore_atomic_funcs.h" // _Py_atomic_int_get()
 #include "pycore_bitutils.h"     // _Py_bswap32()
 #include "pycore_bytesobject.h"  // _PyBytes_Find()
-#include "pycore_ceval.h"        // _PyEval_AddPendingCall
 #include "pycore_compile.h"      // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble, _PyCompile_CleanDoc
+#include "pycore_ceval.h"        // _PyEval_AddPendingCall
 #include "pycore_fileutils.h"    // _Py_normpath
 #include "pycore_frame.h"        // _PyInterpreterFrame
 #include "pycore_gc.h"           // PyGC_Head
 #include "pycore_hashtable.h"    // _Py_hashtable_new()
 #include "pycore_initconfig.h"   // _Py_GetConfigsAsDict()
 #include "pycore_interp.h"       // _PyInterpreterState_GetConfigCopy()
-#include "pycore_interp_id.h"    // _PyInterpreterID_LookUp()
-#include "pycore_object.h"        // _PyObject_IsFreed()
+#include "pycore_object.h"       // _PyObject_IsFreed()
 #include "pycore_pathconfig.h"   // _PyPathConfig_ClearGlobal()
 #include "pycore_pyerrors.h"     // _Py_UTF8_Edit_Cost()
 #include "pycore_pystate.h"      // _PyThreadState_GET()
 
 #include "frameobject.h"
+#include "interpreteridobject.h" // PyInterpreterID_LookUp()
 #include "osdefs.h"              // MAXPATHLEN
 
 #include "clinic/_testinternalcapi.c.h"
 
 #ifdef MS_WINDOWS
-#  include <winsock2.h>           // struct timeval
+#  include <winsock2.h>          // struct timeval
 #endif
 
 
@@ -1083,7 +1083,7 @@ pending_identify(PyObject *self, PyObject *args)
     if (!PyArg_ParseTuple(args, "O:pending_identify", &interpid)) {
         return NULL;
     }
-    PyInterpreterState *interp = _PyInterpreterID_LookUp(interpid);
+    PyInterpreterState *interp = PyInterpreterID_LookUp(interpid);
     if (interp == NULL) {
         if (!PyErr_Occurred()) {
             PyErr_SetString(PyExc_ValueError, "interpreter not found");
@@ -1433,7 +1433,7 @@ test_atexit(PyObject *self, PyObject *Py_UNUSED(args))
     PyThreadState *tstate = Py_NewInterpreter();
 
     struct atexit_data data = {0};
-    int res = _Py_AtExit(tstate->interp, callback, (void *)&data);
+    int res = PyUnstable_AtExit(tstate->interp, callback, (void *)&data);
     Py_EndInterpreter(tstate);
     PyThreadState_Swap(oldts);
     if (res < 0) {

Modules/_xxinterpchannelsmodule.c

@@ -1,13 +1,8 @@
 /* interpreters module */
 /* low-level access to interpreter primitives */
 
-#ifndef Py_BUILD_CORE_BUILTIN
-#  define Py_BUILD_CORE_MODULE 1
-#endif
-
 #include "Python.h"
-#include "pycore_atexit.h"        // _Py_AtExit()
-#include "pycore_interp_id.h"     // _PyInterpreterState_GetIDObject()
+#include "interpreteridobject.h"
 
 
 /*
@@ -2140,7 +2135,7 @@ channel_list_interpreters(PyObject *self, PyObject *args, PyObject *kwds)
             goto except;
         }
         if (res) {
-            id_obj = _PyInterpreterState_GetIDObject(interp);
+            id_obj = PyInterpreterState_GetIDObject(interp);
             if (id_obj == NULL) {
                 goto except;
             }
@@ -2407,7 +2402,7 @@ module_exec(PyObject *mod)
 
     // Make sure chnnels drop objects owned by this interpreter
     PyInterpreterState *interp = _get_current_interp();
-    _Py_AtExit(interp, clear_interpreter, (void *)interp);
+    PyUnstable_AtExit(interp, clear_interpreter, (void *)interp);
 
     return 0;
 

Modules/_xxsubinterpretersmodule.c

@@ -1,13 +1,8 @@
 /* interpreters module */
 /* low-level access to interpreter primitives */
 
-#ifndef Py_BUILD_CORE_BUILTIN
-#  define Py_BUILD_CORE_MODULE 1
-#endif
-
 #include "Python.h"
-#include "pycore_interp.h"        // _PyInterpreterState_GetMainModule()
-#include "pycore_interp_id.h"     // _PyInterpreterState_GetIDObject()
+#include "interpreteridobject.h"
 
 
 #define MODULE_NAME "_xxsubinterpreters"
@@ -400,7 +395,7 @@ _run_script(PyInterpreterState *interp, const char *codestr,
             _sharedns *shared, _sharedexception *sharedexc)
 {
     PyObject *excval = NULL;
-    PyObject *main_mod = _PyInterpreterState_GetMainModule(interp);
+    PyObject *main_mod = PyUnstable_InterpreterState_GetMainModule(interp);
     if (main_mod == NULL) {
         goto error;
     }
@@ -531,7 +526,7 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds)
     }
     assert(tstate != NULL);
     PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate);
-    PyObject *idobj = _PyInterpreterState_GetIDObject(interp);
+    PyObject *idobj = PyInterpreterState_GetIDObject(interp);
     if (idobj == NULL) {
         // XXX Possible GILState issues?
         save_tstate = PyThreadState_Swap(tstate);
@@ -561,7 +556,7 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds)
     }
 
     // Look up the interpreter.
-    PyInterpreterState *interp = _PyInterpreterID_LookUp(id);
+    PyInterpreterState *interp = PyInterpreterID_LookUp(id);
     if (interp == NULL) {
         return NULL;
     }
@@ -616,7 +611,7 @@ interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored))
 
     interp = PyInterpreterState_Head();
     while (interp != NULL) {
-        id = _PyInterpreterState_GetIDObject(interp);
+        id = PyInterpreterState_GetIDObject(interp);
         if (id == NULL) {
             Py_DECREF(ids);
             return NULL;
@@ -648,7 +643,7 @@ interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored))
     if (interp == NULL) {
         return NULL;
     }
-    return _PyInterpreterState_GetIDObject(interp);
+    return PyInterpreterState_GetIDObject(interp);
 }
 
 PyDoc_STRVAR(get_current_doc,
@@ -662,7 +657,7 @@ interp_get_main(PyObject *self, PyObject *Py_UNUSED(ignored))
 {
     // Currently, 0 is always the main interpreter.
     int64_t id = 0;
-    return _PyInterpreterID_New(id);
+    return PyInterpreterID_New(id);
 }
 
 PyDoc_STRVAR(get_main_doc,
@@ -684,7 +679,7 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds)
     }
 
     // Look up the interpreter.
-    PyInterpreterState *interp = _PyInterpreterID_LookUp(id);
+    PyInterpreterState *interp = PyInterpreterID_LookUp(id);
     if (interp == NULL) {
         return NULL;
     }
@@ -750,7 +745,7 @@ interp_is_running(PyObject *self, PyObject *args, PyObject *kwds)
         return NULL;
     }
 
-    PyInterpreterState *interp = _PyInterpreterID_LookUp(id);
+    PyInterpreterState *interp = PyInterpreterID_LookUp(id);
     if (interp == NULL) {
         return NULL;
     }
@@ -808,7 +803,7 @@ module_exec(PyObject *mod)
     }
 
     // PyInterpreterID
-    if (PyModule_AddType(mod, &_PyInterpreterID_Type) < 0) {
+    if (PyModule_AddType(mod, &PyInterpreterID_Type) < 0) {
         goto error;
     }
 

Modules/atexitmodule.c

@@ -24,8 +24,8 @@ get_atexit_state(void)
 
 
 int
-_Py_AtExit(PyInterpreterState *interp,
-           atexit_datacallbackfunc func, void *data)
+PyUnstable_AtExit(PyInterpreterState *interp,
+                  atexit_datacallbackfunc func, void *data)
 {
     assert(interp == _PyInterpreterState_GET());
     atexit_callback *callback = PyMem_Malloc(sizeof(atexit_callback));

Objects/interpreteridobject.c

@@ -3,7 +3,7 @@
 #include "Python.h"
 #include "pycore_abstract.h"   // _PyIndex_Check()
 #include "pycore_interp.h"     // _PyInterpreterState_LookUpID()
-#include "pycore_interp_id.h"  // _PyInterpreterID_Type
+#include "interpreteridobject.h"
 
 
 typedef struct interpid {
@@ -46,7 +46,7 @@ static int
 interp_id_converter(PyObject *arg, void *ptr)
 {
     int64_t id;
-    if (PyObject_TypeCheck(arg, &_PyInterpreterID_Type)) {
+    if (PyObject_TypeCheck(arg, &PyInterpreterID_Type)) {
         id = ((interpid *)arg)->id;
     }
     else if (_PyIndex_Check(arg)) {
@@ -183,13 +183,13 @@ interpid_richcompare(PyObject *self, PyObject *other, int op)
         Py_RETURN_NOTIMPLEMENTED;
     }
 
-    if (!PyObject_TypeCheck(self, &_PyInterpreterID_Type)) {
+    if (!PyObject_TypeCheck(self, &PyInterpreterID_Type)) {
         Py_RETURN_NOTIMPLEMENTED;
     }
 
     interpid *id = (interpid *)self;
     int equal;
-    if (PyObject_TypeCheck(other, &_PyInterpreterID_Type)) {
+    if (PyObject_TypeCheck(other, &PyInterpreterID_Type)) {
         interpid *otherid = (interpid *)other;
         equal = (id->id == otherid->id);
     }
@@ -224,7 +224,7 @@ interpid_richcompare(PyObject *self, PyObject *other, int op)
 PyDoc_STRVAR(interpid_doc,
 "A interpreter ID identifies a interpreter and may be used as an int.");
 
-PyTypeObject _PyInterpreterID_Type = {
+PyTypeObject PyInterpreterID_Type = {
     PyVarObject_HEAD_INIT(&PyType_Type, 0)
     "InterpreterID",   /* tp_name */
     sizeof(interpid),               /* tp_basicsize */
@@ -265,13 +265,13 @@ PyTypeObject _PyInterpreterID_Type = {
     interpid_new,                   /* tp_new */
 };
 
-PyObject *_PyInterpreterID_New(int64_t id)
+PyObject *PyInterpreterID_New(int64_t id)
 {
-    return (PyObject *)newinterpid(&_PyInterpreterID_Type, id, 0);
+    return (PyObject *)newinterpid(&PyInterpreterID_Type, id, 0);
 }
 
 PyObject *
-_PyInterpreterState_GetIDObject(PyInterpreterState *interp)
+PyInterpreterState_GetIDObject(PyInterpreterState *interp)
 {
     if (_PyInterpreterState_IDInitref(interp) != 0) {
         return NULL;
@@ -280,11 +280,11 @@ _PyInterpreterState_GetIDObject(PyInterpreterState *interp)
     if (id < 0) {
         return NULL;
     }
-    return (PyObject *)newinterpid(&_PyInterpreterID_Type, id, 0);
+    return (PyObject *)newinterpid(&PyInterpreterID_Type, id, 0);
 }
 
 PyInterpreterState *
-_PyInterpreterID_LookUp(PyObject *requested_id)
+PyInterpreterID_LookUp(PyObject *requested_id)
 {
     int64_t id;
     if (!interp_id_converter(requested_id, &id)) {

Objects/object.c

@@ -9,16 +9,16 @@
 #include "pycore_dict.h"          // _PyObject_MakeDictFromInstanceAttributes()
 #include "pycore_floatobject.h"   // _PyFloat_DebugMallocStats()
 #include "pycore_initconfig.h"    // _PyStatus_EXCEPTION()
-#include "pycore_interp_id.h"     // _PyInterpreterID_Type
 #include "pycore_namespace.h"     // _PyNamespace_Type
 #include "pycore_object.h"        // PyAPI_DATA() _Py_SwappedOp definition
 #include "pycore_pyerrors.h"      // _PyErr_Occurred()
 #include "pycore_pymem.h"         // _PyMem_IsPtrFreed()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
 #include "pycore_symtable.h"      // PySTEntry_Type
-#include "pycore_typeobject.h"    // _PyBufferWrapper_Type
 #include "pycore_typevarobject.h" // _PyTypeAlias_Type, _Py_initialize_generic
+#include "pycore_typeobject.h"    // _PyBufferWrapper_Type
 #include "pycore_unionobject.h"   // _PyUnion_Type
+#include "interpreteridobject.h"  // _PyInterpreterID_Type
 
 #ifdef Py_LIMITED_API
    // Prevent recursive call _Py_IncRef() <=> Py_INCREF()
@@ -2072,6 +2072,7 @@ static PyTypeObject* static_types[] = {
     &PyGen_Type,
     &PyGetSetDescr_Type,
     &PyInstanceMethod_Type,
+    &PyInterpreterID_Type,
     &PyListIter_Type,
     &PyListRevIter_Type,
     &PyList_Type,
@@ -2122,7 +2123,6 @@ static PyTypeObject* static_types[] = {
     &_PyHamt_CollisionNode_Type,
     &_PyHamt_Type,
     &_PyLegacyEventHandler_Type,
-    &_PyInterpreterID_Type,
     &_PyLineIterator,
     &_PyManagedBuffer_Type,
     &_PyMemoryIter_Type,

PCbuild/pythoncore.vcxproj

@@ -152,6 +152,7 @@
     <ClInclude Include="..\Include\cpython\genobject.h" />
     <ClInclude Include="..\Include\cpython\import.h" />
     <ClInclude Include="..\Include\cpython\initconfig.h" />
+    <ClInclude Include="..\Include\cpython\interpreteridobject.h" />
     <ClInclude Include="..\Include\cpython\listobject.h" />
     <ClInclude Include="..\Include\cpython\longintrepr.h" />
     <ClInclude Include="..\Include\cpython\longobject.h" />
@@ -237,7 +238,6 @@
     <ClInclude Include="..\Include\internal\pycore_import.h" />
     <ClInclude Include="..\Include\internal\pycore_initconfig.h" />
     <ClInclude Include="..\Include\internal\pycore_interp.h" />
-    <ClInclude Include="..\Include\internal\pycore_interp_id.h" />
     <ClInclude Include="..\Include\internal\pycore_intrinsics.h" />
     <ClInclude Include="..\Include\internal\pycore_list.h" />
     <ClInclude Include="..\Include\internal\pycore_long.h" />
@@ -281,6 +281,7 @@
     <ClInclude Include="..\Include\internal\pycore_unicodeobject_generated.h" />
     <ClInclude Include="..\Include\internal\pycore_warnings.h" />
     <ClInclude Include="..\Include\internal\pycore_weakref.h" />
+    <ClInclude Include="..\Include\interpreteridobject.h" />
     <ClInclude Include="..\Include\intrcheck.h" />
     <ClInclude Include="..\Include\iterobject.h" />
     <ClInclude Include="..\Include\listobject.h" />

PCbuild/pythoncore.vcxproj.filters

@@ -312,6 +312,9 @@
     <ClInclude Include="..\Include\pyhash.h">
       <Filter>Include</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\interpreteridobject.h">
+      <Filter>Include</Filter>
+    </ClInclude>
     <ClInclude Include="..\Modules\hashtable.h">
       <Filter>Modules</Filter>
     </ClInclude>
@@ -459,6 +462,9 @@
     <ClInclude Include="..\Include\cpython\genobject.h">
       <Filter>Include</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\cpython\interpreteridobject.h">
+      <Filter>Include\cpython</Filter>
+    </ClInclude>
     <ClInclude Include="..\Include\cpython\pythonrun.h">
       <Filter>Include\cpython</Filter>
     </ClInclude>
@@ -612,9 +618,6 @@
     <ClInclude Include="..\Include\internal\pycore_interp.h">
       <Filter>Include\internal</Filter>
     </ClInclude>
-    <ClInclude Include="..\Include\internal\pycore_interp_id.h">
-      <Filter>Include\internal</Filter>
-    </ClInclude>
     <ClInclude Include="..\Include\internal\pycore_intrinsics.h">
       <Filter>Include\cpython</Filter>
     </ClInclude>

Python/pystate.c

@@ -1133,7 +1133,7 @@ _PyInterpreterState_RequireIDRef(PyInterpreterState *interp, int required)
 }
 
 PyObject *
-_PyInterpreterState_GetMainModule(PyInterpreterState *interp)
+PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *interp)
 {
     PyObject *modules = _PyImport_GetModules(interp);
     if (modules == NULL) {

Tools/c-analyzer/cpython/_parser.py

@@ -227,6 +227,7 @@ def clean_lines(text):
 Include/cpython/fileutils.h	Py_CPYTHON_FILEUTILS_H	1
 Include/cpython/frameobject.h	Py_CPYTHON_FRAMEOBJECT_H	1
 Include/cpython/import.h	Py_CPYTHON_IMPORT_H	1
+Include/cpython/interpreteridobject.h	Py_CPYTHON_INTERPRETERIDOBJECT_H	1
 Include/cpython/listobject.h	Py_CPYTHON_LISTOBJECT_H	1
 Include/cpython/methodobject.h	Py_CPYTHON_METHODOBJECT_H	1
 Include/cpython/object.h	Py_CPYTHON_OBJECT_H	1

Tools/c-analyzer/cpython/globals-to-fix.tsv

@@ -54,7 +54,7 @@ Objects/genobject.c	-	_PyAsyncGenASend_Type	-
 Objects/genobject.c	-	_PyAsyncGenAThrow_Type	-
 Objects/genobject.c	-	_PyAsyncGenWrappedValue_Type	-
 Objects/genobject.c	-	_PyCoroWrapper_Type	-
-Objects/interpreteridobject.c	-	_PyInterpreterID_Type	-
+Objects/interpreteridobject.c	-	PyInterpreterID_Type	-
 Objects/iterobject.c	-	PyCallIter_Type	-
 Objects/iterobject.c	-	PySeqIter_Type	-
 Objects/listobject.c	-	PyListIter_Type	-