bpo-32604: Expose the subinterpreters C-API in a "private" stdlib module. (gh-1748)

The module is primarily intended for internal use in the test suite.  Building the module under Windows will come in a follow-up PR.

Include/internal/pystate.h

@@ -65,6 +65,79 @@ PyAPI_FUNC(_PyInitError) _PyPathConfig_Calculate(
 PyAPI_FUNC(void) _PyPathConfig_Clear(_PyPathConfig *config);
 
 
+/* interpreter state */
+
+PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(PY_INT64_T);
+
+
+/* cross-interpreter data */
+
+struct _xid;
+
+// _PyCrossInterpreterData is similar to Py_buffer as an effectively
+// opaque struct that holds data outside the object machinery.  This
+// is necessary to pass between interpreters in the same process.
+typedef struct _xid {
+    // data is the cross-interpreter-safe derivation of a Python object
+    // (see _PyObject_GetCrossInterpreterData).  It will be NULL if the
+    // new_object func (below) encodes the data.
+    void *data;
+    // obj is the Python object from which the data was derived.  This
+    // is non-NULL only if the data remains bound to the object in some
+    // way, such that the object must be "released" (via a decref) when
+    // the data is released.  In that case it is automatically
+    // incref'ed (to match the automatic decref when releaed).
+    PyObject *obj;
+    // interp is the ID of the owning interpreter of the original
+    // object.  It corresponds to the active interpreter when
+    // _PyObject_GetCrossInterpreterData() was called.  This should only
+    // be set by the cross-interpreter machinery.
+    //
+    // We use the ID rather than the PyInterpreterState to avoid issues
+    // with deleted interpreters.
+    int64_t interp;
+    // new_object is a function that returns a new object in the current
+    // interpreter given the data.  The resulting object (a new
+    // reference) will be equivalent to the original object.  This field
+    // is required.
+    PyObject *(*new_object)(struct _xid *);
+    // free is called when the data is released.  If it is NULL then
+    // nothing will be done to free the data.  For some types this is
+    // okay (e.g. bytes) and for those types this field should be set
+    // to NULL.  However, for most the data was allocated just for
+    // cross-interpreter use, so it must be freed when
+    // _PyCrossInterpreterData_Release is called or the memory will
+    // leak.  In that case, at the very least this field should be set
+    // to PyMem_RawFree (the default if not explicitly set to NULL).
+    // The call will happen with the original interpreter activated.
+    void (*free)(void *);
+} _PyCrossInterpreterData;
+
+typedef int (*crossinterpdatafunc)(PyObject *, _PyCrossInterpreterData *);
+PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *);
+
+PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *);
+PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *);
+PyAPI_FUNC(void) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *);
+
+/* cross-interpreter data registry */
+
+/* For now we use a global registry of shareable classes.  An
+   alternative would be to add a tp_* slot for a class's
+   crossinterpdatafunc. It would be simpler and more efficient. */
+
+PyAPI_FUNC(int) _PyCrossInterpreterData_Register_Class(PyTypeObject *, crossinterpdatafunc);
+PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *);
+
+struct _xidregitem;
+
+struct _xidregitem {
+    PyTypeObject *cls;
+    crossinterpdatafunc getdata;
+    struct _xidregitem *next;
+};
+
+
 /* Full Python runtime state */
 
 typedef struct pyruntimestate {
@@ -86,6 +159,11 @@ typedef struct pyruntimestate {
            using a Python int. */
         int64_t next_id;
     } interpreters;
+    // XXX Remove this field once we have a tp_* slot.
+    struct _xidregistry {
+        PyThread_type_lock mutex;
+        struct _xidregitem *head;
+    } xidregistry;
 
 #define NEXITFUNCS 32
     void (*exitfuncs[NEXITFUNCS])(void);

Misc/NEWS.d/next/Tests/2018-01-26-21-29-09.bpo-32604.7iazNx.rst

@@ -0,0 +1,4 @@
+Add a new "_xxsubinterpreters" extension module that exposes the existing
+subinterpreter C-API and a new cross-interpreter data sharing mechanism. The
+module is primarily intended for more thorough testing of the existing
+subinterpreter support.

Python/pystate.c

@@ -54,8 +54,13 @@ _PyRuntimeState_Init_impl(_PyRuntimeState *runtime)
     if (runtime->interpreters.mutex == NULL) {
         return _Py_INIT_ERR("Can't initialize threads for interpreter");
     }
-
     runtime->interpreters.next_id = -1;
+
+    runtime->xidregistry.mutex = PyThread_allocate_lock();
+    if (runtime->xidregistry.mutex == NULL) {
+        return _Py_INIT_ERR("Can't initialize threads for cross-interpreter data registry");
+    }
+
     return _Py_INIT_OK();
 }
 
@@ -166,6 +171,7 @@ PyInterpreterState_New(void)
         /* overflow or Py_Initialize() not called! */
         PyErr_SetString(PyExc_RuntimeError,
                         "failed to get an interpreter ID");
+        /* XXX deallocate! */
         interp = NULL;
     } else {
         interp->id = _PyRuntime.interpreters.next_id;
@@ -256,6 +262,28 @@ PyInterpreterState_GetID(PyInterpreterState *interp)
 }
 
 
+PyInterpreterState *
+_PyInterpreterState_LookUpID(PY_INT64_T requested_id)
+{
+    if (requested_id < 0)
+        goto error;
+
+    PyInterpreterState *interp = PyInterpreterState_Head();
+    while (interp != NULL) {
+        PY_INT64_T id = PyInterpreterState_GetID(interp);
+        if (id < 0)
+            return NULL;
+        if (requested_id == id)
+            return interp;
+        interp = PyInterpreterState_Next(interp);
+    }
+
+error:
+    PyErr_Format(PyExc_RuntimeError,
+                 "unrecognized interpreter ID %lld", requested_id);
+    return NULL;
+}
+
 /* Default implementation for _PyThreadState_GetFrame */
 static struct _frame *
 threadstate_getframe(PyThreadState *self)
@@ -1024,6 +1052,251 @@ PyGILState_Release(PyGILState_STATE oldstate)
 }
 
 
+/**************************/
+/* cross-interpreter data */
+/**************************/
+
+/* cross-interpreter data */
+
+crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *);
+
+/* This is a separate func from _PyCrossInterpreterData_Lookup in order
+   to keep the registry code separate. */
+static crossinterpdatafunc
+_lookup_getdata(PyObject *obj)
+{
+    crossinterpdatafunc getdata = _PyCrossInterpreterData_Lookup(obj);
+    if (getdata == NULL && PyErr_Occurred() == 0)
+        PyErr_Format(PyExc_ValueError,
+                     "%S does not support cross-interpreter data", obj);
+    return getdata;
+}
+
+int
+_PyObject_CheckCrossInterpreterData(PyObject *obj)
+{
+    crossinterpdatafunc getdata = _lookup_getdata(obj);
+    if (getdata == NULL) {
+        return -1;
+    }
+    return 0;
+}
+
+static int
+_check_xidata(_PyCrossInterpreterData *data)
+{
+    // data->data can be anything, including NULL, so we don't check it.
+
+    // data->obj may be NULL, so we don't check it.
+
+    if (data->interp < 0) {
+        PyErr_SetString(PyExc_SystemError, "missing interp");
+        return -1;
+    }
+
+    if (data->new_object == NULL) {
+        PyErr_SetString(PyExc_SystemError, "missing new_object func");
+        return -1;
+    }
+
+    // data->free may be NULL, so we don't check it.
+
+    return 0;
+}
+
+int
+_PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data)
+{
+    PyThreadState *tstate = PyThreadState_Get();
+    // PyThreadState_Get() aborts if lookup fails, so we don't need
+    // to check the result for NULL.
+    PyInterpreterState *interp = tstate->interp;
+
+    // Reset data before re-populating.
+    *data = (_PyCrossInterpreterData){0};
+    data->free = PyMem_RawFree;  // Set a default that may be overridden.
+
+    // Call the "getdata" func for the object.
+    Py_INCREF(obj);
+    crossinterpdatafunc getdata = _lookup_getdata(obj);
+    if (getdata == NULL) {
+        Py_DECREF(obj);
+        return -1;
+    }
+    int res = getdata(obj, data);
+    Py_DECREF(obj);
+    if (res != 0) {
+        return -1;
+    }
+
+    // Fill in the blanks and validate the result.
+    Py_XINCREF(data->obj);
+    data->interp = interp->id;
+    if (_check_xidata(data) != 0) {
+        _PyCrossInterpreterData_Release(data);
+        return -1;
+    }
+
+    return 0;
+}
+
+void
+_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data)
+{
+    if (data->data == NULL && data->obj == NULL) {
+        // Nothing to release!
+        return;
+    }
+
+    // Switch to the original interpreter.
+    PyInterpreterState *interp = _PyInterpreterState_LookUpID(data->interp);
+    if (interp == NULL) {
+        // The intepreter was already destroyed.
+        if (data->free != NULL) {
+            // XXX Someone leaked some memory...
+        }
+        return;
+    }
+    PyThreadState *tstate = PyInterpreterState_ThreadHead(interp);
+    PyThreadState *save_tstate = PyThreadState_Swap(tstate);
+
+    // "Release" the data and/or the object.
+    if (data->free != NULL) {
+        data->free(data->data);
+    }
+    Py_XDECREF(data->obj);
+
+    // Switch back.
+    if (save_tstate != NULL)
+        PyThreadState_Swap(save_tstate);
+}
+
+PyObject *
+_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data)
+{
+    return data->new_object(data);
+}
+
+/* registry of {type -> crossinterpdatafunc} */
+
+/* For now we use a global registry of shareable classes.  An
+   alternative would be to add a tp_* slot for a class's
+   crossinterpdatafunc. It would be simpler and more efficient. */
+
+static int
+_register_xidata(PyTypeObject *cls, crossinterpdatafunc getdata)
+{
+    // Note that we effectively replace already registered classes
+    // rather than failing.
+    struct _xidregitem *newhead = PyMem_RawMalloc(sizeof(struct _xidregitem));
+    if (newhead == NULL)
+        return -1;
+    newhead->cls = cls;
+    newhead->getdata = getdata;
+    newhead->next = _PyRuntime.xidregistry.head;
+    _PyRuntime.xidregistry.head = newhead;
+    return 0;
+}
+
+static void _register_builtins_for_crossinterpreter_data(void);
+
+int
+_PyCrossInterpreterData_Register_Class(PyTypeObject *cls,
+                                       crossinterpdatafunc getdata)
+{
+    if (!PyType_Check(cls)) {
+        PyErr_Format(PyExc_ValueError, "only classes may be registered");
+        return -1;
+    }
+    if (getdata == NULL) {
+        PyErr_Format(PyExc_ValueError, "missing 'getdata' func");
+        return -1;
+    }
+
+    // Make sure the class isn't ever deallocated.
+    Py_INCREF((PyObject *)cls);
+
+    PyThread_acquire_lock(_PyRuntime.xidregistry.mutex, WAIT_LOCK);
+    if (_PyRuntime.xidregistry.head == NULL) {
+        _register_builtins_for_crossinterpreter_data();
+    }
+    int res = _register_xidata(cls, getdata);
+    PyThread_release_lock(_PyRuntime.xidregistry.mutex);
+    return res;
+}
+
+crossinterpdatafunc
+_PyCrossInterpreterData_Lookup(PyObject *obj)
+{
+    PyObject *cls = PyObject_Type(obj);
+    crossinterpdatafunc getdata = NULL;
+    PyThread_acquire_lock(_PyRuntime.xidregistry.mutex, WAIT_LOCK);
+    struct _xidregitem *cur = _PyRuntime.xidregistry.head;
+    if (cur == NULL) {
+        _register_builtins_for_crossinterpreter_data();
+        cur = _PyRuntime.xidregistry.head;
+    }
+    for(; cur != NULL; cur = cur->next) {
+        if (cur->cls == (PyTypeObject *)cls) {
+            getdata = cur->getdata;
+            break;
+        }
+    }
+    PyThread_release_lock(_PyRuntime.xidregistry.mutex);
+    return getdata;
+}
+
+/* cross-interpreter data for builtin types */
+
+static PyObject *
+_new_bytes_object(_PyCrossInterpreterData *data)
+{
+    return PyBytes_FromString((char *)(data->data));
+}
+
+static int
+_bytes_shared(PyObject *obj, _PyCrossInterpreterData *data)
+{
+    data->data = (void *)(PyBytes_AS_STRING(obj));
+    data->obj = obj;  // Will be "released" (decref'ed) when data released.
+    data->new_object = _new_bytes_object;
+    data->free = NULL;  // Do not free the data (it belongs to the object).
+    return 0;
+}
+
+static PyObject *
+_new_none_object(_PyCrossInterpreterData *data)
+{
+    // XXX Singleton refcounts are problematic across interpreters...
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+
+static int
+_none_shared(PyObject *obj, _PyCrossInterpreterData *data)
+{
+    data->data = NULL;
+    // data->obj remains NULL
+    data->new_object = _new_none_object;
+    data->free = NULL;  // There is nothing to free.
+    return 0;
+}
+
+static void
+_register_builtins_for_crossinterpreter_data(void)
+{
+    // None
+    if (_register_xidata((PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) {
+        Py_FatalError("could not register None for cross-interpreter sharing");
+    }
+
+    // bytes
+    if (_register_xidata(&PyBytes_Type, _bytes_shared) != 0) {
+        Py_FatalError("could not register bytes for cross-interpreter sharing");
+    }
+}
+
+
 #ifdef __cplusplus
 }
 #endif

setup.py

@@ -755,6 +755,10 @@ def detect_modules(self):
             ['_xxtestfuzz/_xxtestfuzz.c', '_xxtestfuzz/fuzzer.c'])
         )
 
+        # Python interface to subinterpreter C-API.
+        exts.append(Extension('_xxsubinterpreters', ['_xxsubinterpretersmodule.c'],
+                              define_macros=[('Py_BUILD_CORE', '')]))
+
         #
         # Here ends the simple stuff.  From here on, modules need certain
         # libraries, are platform-specific, or present other surprises.