gh-76785: More Fixes for test.support.interpreters (gh-113012)

This brings the module (along with the associated extension modules) mostly in sync with PEP 734.  There are only a few small things to wrap up.

Lib/test/support/interpreters/queues.py

@@ -3,13 +3,11 @@
 import queue
 import time
 import weakref
-import _xxinterpchannels as _channels
-import _xxinterpchannels as _queues
+import _xxinterpqueues as _queues
 
 # aliases:
-from _xxinterpchannels import (
-    ChannelError as QueueError,
-    ChannelNotFoundError as QueueNotFoundError,
+from _xxinterpqueues import (
+    QueueError, QueueNotFoundError,
 )
 
 __all__ = [
@@ -19,14 +17,27 @@ __all__ = [
 ]
 
 
+class QueueEmpty(_queues.QueueEmpty, queue.Empty):
+    """Raised from get_nowait() when the queue is empty.
+
+    It is also raised from get() if it times out.
+    """
+
+
+class QueueFull(_queues.QueueFull, queue.Full):
+    """Raised from put_nowait() when the queue is full.
+
+    It is also raised from put() if it times out.
+    """
+
+
 def create(maxsize=0):
     """Return a new cross-interpreter queue.
 
     The queue may be used to pass data safely between interpreters.
     """
-    # XXX honor maxsize
-    qid = _queues.create()
-    return Queue._with_maxsize(qid, maxsize)
+    qid = _queues.create(maxsize)
+    return Queue(qid)
 
 
 def list_all():
@@ -35,53 +46,37 @@ def list_all():
             for qid in _queues.list_all()]
 
 
-class QueueEmpty(queue.Empty):
-    """Raised from get_nowait() when the queue is empty.
-
-    It is also raised from get() if it times out.
-    """
-
-
-class QueueFull(queue.Full):
-    """Raised from put_nowait() when the queue is full.
-
-    It is also raised from put() if it times out.
-    """
-
 
 _known_queues = weakref.WeakValueDictionary()
 
 class Queue:
     """A cross-interpreter queue."""
 
-    @classmethod
-    def _with_maxsize(cls, id, maxsize):
-        if not isinstance(maxsize, int):
-            raise TypeError(f'maxsize must be an int, got {maxsize!r}')
-        elif maxsize < 0:
-            maxsize = 0
-        else:
-            maxsize = int(maxsize)
-        self = cls(id)
-        self._maxsize = maxsize
-        return self
-
     def __new__(cls, id, /):
         # There is only one instance for any given ID.
         if isinstance(id, int):
-            id = _channels._channel_id(id, force=False)
-        elif not isinstance(id, _channels.ChannelID):
+            id = int(id)
+        else:
             raise TypeError(f'id must be an int, got {id!r}')
-        key = int(id)
         try:
-            self = _known_queues[key]
+            self = _known_queues[id]
         except KeyError:
             self = super().__new__(cls)
             self._id = id
-            self._maxsize = 0
-            _known_queues[key] = self
+            _known_queues[id] = self
+            _queues.bind(id)
         return self
 
+    def __del__(self):
+        try:
+            _queues.release(self._id)
+        except QueueNotFoundError:
+            pass
+        try:
+            del _known_queues[self._id]
+        except KeyError:
+            pass
+
     def __repr__(self):
         return f'{type(self).__name__}({self.id})'
 
@@ -90,39 +85,58 @@ class Queue:
 
     @property
     def id(self):
-        return int(self._id)
+        return self._id
 
     @property
     def maxsize(self):
-        return self._maxsize
-
-    @property
-    def _info(self):
-        return _channels.get_info(self._id)
+        try:
+            return self._maxsize
+        except AttributeError:
+            self._maxsize = _queues.get_maxsize(self._id)
+            return self._maxsize
 
     def empty(self):
-        return self._info.count == 0
+        return self.qsize() == 0
 
     def full(self):
-        if self._maxsize <= 0:
-            return False
-        return self._info.count >= self._maxsize
+        return _queues.is_full(self._id)
 
     def qsize(self):
-        return self._info.count
+        return _queues.get_count(self._id)
 
-    def put(self, obj, timeout=None):
-        # XXX block if full
-        _channels.send(self._id, obj, blocking=False)
+    def put(self, obj, timeout=None, *,
+            _delay=10 / 1000,  # 10 milliseconds
+            ):
+        """Add the object to the queue.
+
+        This blocks while the queue is full.
+        """
+        if timeout is not None:
+            timeout = int(timeout)
+            if timeout < 0:
+                raise ValueError(f'timeout value must be non-negative')
+            end = time.time() + timeout
+        while True:
+            try:
+                _queues.put(self._id, obj)
+            except _queues.QueueFull as exc:
+                if timeout is not None and time.time() >= end:
+                    exc.__class__ = QueueFull
+                    raise  # re-raise
+                time.sleep(_delay)
+            else:
+                break
 
     def put_nowait(self, obj):
-        # XXX raise QueueFull if full
-        return _channels.send(self._id, obj, blocking=False)
+        try:
+            return _queues.put(self._id, obj)
+        except _queues.QueueFull as exc:
+            exc.__class__ = QueueFull
+            raise  # re-raise
 
     def get(self, timeout=None, *,
-             _sentinel=object(),
-             _delay=10 / 1000,  # 10 milliseconds
-             ):
+            _delay=10 / 1000,  # 10 milliseconds
+            ):
         """Return the next object from the queue.
 
         This blocks while the queue is empty.
@@ -132,25 +146,27 @@ class Queue:
             if timeout < 0:
                 raise ValueError(f'timeout value must be non-negative')
             end = time.time() + timeout
-        obj = _channels.recv(self._id, _sentinel)
-        while obj is _sentinel:
-            time.sleep(_delay)
-            if timeout is not None and time.time() >= end:
-                raise QueueEmpty
-            obj = _channels.recv(self._id, _sentinel)
+        while True:
+            try:
+                return _queues.get(self._id)
+            except _queues.QueueEmpty as exc:
+                if timeout is not None and time.time() >= end:
+                    exc.__class__ = QueueEmpty
+                    raise  # re-raise
+                time.sleep(_delay)
         return obj
 
-    def get_nowait(self, *, _sentinel=object()):
+    def get_nowait(self):
         """Return the next object from the channel.
 
         If the queue is empty then raise QueueEmpty.  Otherwise this
         is the same as get().
         """
-        obj = _channels.recv(self._id, _sentinel)
-        if obj is _sentinel:
-            raise QueueEmpty
-        return obj
+        try:
+            return _queues.get(self._id)
+        except _queues.QueueEmpty as exc:
+            exc.__class__ = QueueEmpty
+            raise  # re-raise
 
 
-# XXX add this:
-#_channels._register_queue_type(Queue)
+_queues._register_queue_type(Queue)

Lib/test/test_interpreters/test_queues.py

@@ -5,13 +5,21 @@ import time
 
 from test.support import import_helper
 # Raise SkipTest if subinterpreters not supported.
-import_helper.import_module('_xxinterpchannels')
-#import_helper.import_module('_xxinterpqueues')
+_queues = import_helper.import_module('_xxinterpqueues')
 from test.support import interpreters
 from test.support.interpreters import queues
 from .utils import _run_output, TestBase
 
 
+class TestBase(TestBase):
+    def tearDown(self):
+        for qid in _queues.list_all():
+            try:
+                _queues.destroy(qid)
+            except Exception:
+                pass
+
+
 class QueueTests(TestBase):
 
     def test_create(self):
@@ -32,20 +40,47 @@ class QueueTests(TestBase):
             self.assertEqual(queue.maxsize, 0)
 
         with self.subTest('negative maxsize'):
-            queue = queues.create(-1)
-            self.assertEqual(queue.maxsize, 0)
+            queue = queues.create(-10)
+            self.assertEqual(queue.maxsize, -10)
 
         with self.subTest('bad maxsize'):
             with self.assertRaises(TypeError):
                 queues.create('1')
 
-    @unittest.expectedFailure
     def test_shareable(self):
         queue1 = queues.create()
-        queue2 = queues.create()
-        queue1.put(queue2)
-        queue3 = queue1.get()
-        self.assertIs(queue3, queue1)
+
+        interp = interpreters.create()
+        interp.exec_sync(dedent(f"""
+            from test.support.interpreters import queues
+            queue1 = queues.Queue({queue1.id})
+            """));
+
+        with self.subTest('same interpreter'):
+            queue2 = queues.create()
+            queue1.put(queue2)
+            queue3 = queue1.get()
+            self.assertIs(queue3, queue2)
+
+        with self.subTest('from current interpreter'):
+            queue4 = queues.create()
+            queue1.put(queue4)
+            out = _run_output(interp, dedent("""
+                queue4 = queue1.get()
+                print(queue4.id)
+                """))
+            qid = int(out)
+            self.assertEqual(qid, queue4.id)
+
+        with self.subTest('from subinterpreter'):
+            out = _run_output(interp, dedent("""
+                queue5 = queues.create()
+                queue1.put(queue5)
+                print(queue5.id)
+                """))
+            qid = int(out)
+            queue5 = queue1.get()
+            self.assertEqual(queue5.id, qid)
 
     def test_id_type(self):
         queue = queues.create()
@@ -137,7 +172,6 @@ class TestQueueOps(TestBase):
 
         self.assertEqual(actual, expected)
 
-    @unittest.expectedFailure
     def test_put_timeout(self):
         queue = queues.create(2)
         queue.put(None)
@@ -147,7 +181,6 @@ class TestQueueOps(TestBase):
         queue.get()
         queue.put(None)
 
-    @unittest.expectedFailure
     def test_put_nowait(self):
         queue = queues.create(2)
         queue.put_nowait(None)
@@ -179,31 +212,64 @@ class TestQueueOps(TestBase):
             assert obj is not orig, 'expected: obj is not orig'
             """))
 
-    @unittest.expectedFailure
     def test_put_get_different_interpreters(self):
+        interp = interpreters.create()
         queue1 = queues.create()
         queue2 = queues.create()
+        self.assertEqual(len(queues.list_all()), 2)
+
         obj1 = b'spam'
         queue1.put(obj1)
+
         out = _run_output(
-            interpreters.create(),
+            interp,
             dedent(f"""
-                import test.support.interpreters.queue as queues
+                from test.support.interpreters import queues
                 queue1 = queues.Queue({queue1.id})
                 queue2 = queues.Queue({queue2.id})
+                assert queue1.qsize() == 1, 'expected: queue1.qsize() == 1'
                 obj = queue1.get()
+                assert queue1.qsize() == 0, 'expected: queue1.qsize() == 0'
                 assert obj == b'spam', 'expected: obj == obj1'
                 # When going to another interpreter we get a copy.
                 assert id(obj) != {id(obj1)}, 'expected: obj is not obj1'
                 obj2 = b'eggs'
                 print(id(obj2))
+                assert queue2.qsize() == 0, 'expected: queue2.qsize() == 0'
                 queue2.put(obj2)
+                assert queue2.qsize() == 1, 'expected: queue2.qsize() == 1'
                 """))
-        obj2 = queue2.get()
+        self.assertEqual(len(queues.list_all()), 2)
+        self.assertEqual(queue1.qsize(), 0)
+        self.assertEqual(queue2.qsize(), 1)
 
+        obj2 = queue2.get()
         self.assertEqual(obj2, b'eggs')
         self.assertNotEqual(id(obj2), int(out))
 
+    def test_put_cleared_with_subinterpreter(self):
+        interp = interpreters.create()
+        queue = queues.create()
+
+        out = _run_output(
+            interp,
+            dedent(f"""
+                from test.support.interpreters import queues
+                queue = queues.Queue({queue.id})
+                obj1 = b'spam'
+                obj2 = b'eggs'
+                queue.put(obj1)
+                queue.put(obj2)
+                """))
+        self.assertEqual(queue.qsize(), 2)
+
+        obj1 = queue.get()
+        self.assertEqual(obj1, b'spam')
+        self.assertEqual(queue.qsize(), 1)
+
+        del interp
+        self.assertEqual(queue.qsize(), 0)
+
     def test_put_get_different_threads(self):
         queue1 = queues.create()
         queue2 = queues.create()

Modules/Setup

@@ -273,6 +273,7 @@ PYTHONPATH=$(COREPYTHONPATH)
 
 #_xxsubinterpreters _xxsubinterpretersmodule.c
 #_xxinterpchannels _xxinterpchannelsmodule.c
+#_xxinterpqueues _xxinterpqueuesmodule.c
 #_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c
 #_testbuffer _testbuffer.c
 #_testinternalcapi _testinternalcapi.c

Modules/Setup.stdlib.in

@@ -41,8 +41,11 @@
 @MODULE__QUEUE_TRUE@_queue _queuemodule.c
 @MODULE__RANDOM_TRUE@_random _randommodule.c
 @MODULE__STRUCT_TRUE@_struct _struct.c
+
+# build supports subinterpreters
 @MODULE__XXSUBINTERPRETERS_TRUE@_xxsubinterpreters _xxsubinterpretersmodule.c
 @MODULE__XXINTERPCHANNELS_TRUE@_xxinterpchannels _xxinterpchannelsmodule.c
+@MODULE__XXINTERPQUEUES_TRUE@_xxinterpqueues _xxinterpqueuesmodule.c
 @MODULE__ZONEINFO_TRUE@_zoneinfo _zoneinfo.c
 
 # needs libm

Modules/_xxinterpchannelsmodule.c

@@ -2629,10 +2629,11 @@ _get_current_channelend_type(int end)
         cls = state->recv_channel_type;
     }
     if (cls == NULL) {
-        PyObject *highlevel = PyImport_ImportModule("interpreters");
+        // Force the module to be loaded, to register the type.
+        PyObject *highlevel = PyImport_ImportModule("interpreters.channel");
         if (highlevel == NULL) {
             PyErr_Clear();
-            highlevel = PyImport_ImportModule("test.support.interpreters");
+            highlevel = PyImport_ImportModule("test.support.interpreters.channel");
             if (highlevel == NULL) {
                 return NULL;
             }

PC/config.c

@@ -37,6 +37,7 @@ extern PyObject* PyInit__weakref(void);
 extern PyObject* PyInit_xxsubtype(void);
 extern PyObject* PyInit__xxsubinterpreters(void);
 extern PyObject* PyInit__xxinterpchannels(void);
+extern PyObject* PyInit__xxinterpqueues(void);
 extern PyObject* PyInit__random(void);
 extern PyObject* PyInit_itertools(void);
 extern PyObject* PyInit__collections(void);
@@ -142,6 +143,7 @@ struct _inittab _PyImport_Inittab[] = {
     {"xxsubtype", PyInit_xxsubtype},
     {"_xxsubinterpreters", PyInit__xxsubinterpreters},
     {"_xxinterpchannels", PyInit__xxinterpchannels},
+    {"_xxinterpqueues", PyInit__xxinterpqueues},
 #ifdef _Py_HAVE_ZLIB
     {"zlib", PyInit_zlib},
 #endif

PCbuild/pythoncore.vcxproj

@@ -458,6 +458,7 @@
     <ClCompile Include="..\Modules\xxsubtype.c" />
     <ClCompile Include="..\Modules\_xxsubinterpretersmodule.c" />
     <ClCompile Include="..\Modules\_xxinterpchannelsmodule.c" />
+    <ClCompile Include="..\Modules\_xxinterpqueuesmodule.c" />
     <ClCompile Include="..\Modules\_io\fileio.c" />
     <ClCompile Include="..\Modules\_io\bytesio.c" />
     <ClCompile Include="..\Modules\_io\stringio.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -1505,6 +1505,9 @@
     <ClCompile Include="..\Modules\_xxinterpchannelsmodule.c">
       <Filter>Modules</Filter>
     </ClCompile>
+    <ClCompile Include="..\Modules\_xxinterpqueuesmodule.c">
+      <Filter>Modules</Filter>
+    </ClCompile>
     <ClCompile Include="..\Parser\string_parser.c">
       <Filter>Parser</Filter>
     </ClCompile>

Tools/build/generate_stdlib_module_names.py

@@ -36,6 +36,7 @@ IGNORE = {
     '_testsinglephase',
     '_xxsubinterpreters',
     '_xxinterpchannels',
+    '_xxinterpqueues',
     '_xxtestfuzz',
     'idlelib.idle_test',
     'test',

Tools/c-analyzer/cpython/ignored.tsv

@@ -165,6 +165,7 @@ Python/pylifecycle.c	fatal_error	reentrant	-
 
 # explicitly protected, internal-only
 Modules/_xxinterpchannelsmodule.c	-	_globals	-
+Modules/_xxinterpqueuesmodule.c	-	_globals	-
 
 # set once during module init
 Modules/_decimal/_decimal.c	-	minalloc_is_set	-

configure

@@ -769,6 +769,8 @@ MODULE__MULTIPROCESSING_FALSE
 MODULE__MULTIPROCESSING_TRUE
 MODULE__ZONEINFO_FALSE
 MODULE__ZONEINFO_TRUE
+MODULE__XXINTERPQUEUES_FALSE
+MODULE__XXINTERPQUEUES_TRUE
 MODULE__XXINTERPCHANNELS_FALSE
 MODULE__XXINTERPCHANNELS_TRUE
 MODULE__XXSUBINTERPRETERS_FALSE
@@ -28025,6 +28027,7 @@ case $ac_sys_system in #(
     py_cv_module__tkinter=n/a
     py_cv_module__xxsubinterpreters=n/a
     py_cv_module__xxinterpchannels=n/a
+    py_cv_module__xxinterpqueues=n/a
     py_cv_module_grp=n/a
     py_cv_module_pwd=n/a
     py_cv_module_resource=n/a
@@ -28524,6 +28527,28 @@ then :
 
 
 
+fi
+
+
+        if test "$py_cv_module__xxinterpqueues" != "n/a"
+then :
+  py_cv_module__xxinterpqueues=yes
+fi
+   if test "$py_cv_module__xxinterpqueues" = yes; then
+  MODULE__XXINTERPQUEUES_TRUE=
+  MODULE__XXINTERPQUEUES_FALSE='#'
+else
+  MODULE__XXINTERPQUEUES_TRUE='#'
+  MODULE__XXINTERPQUEUES_FALSE=
+fi
+
+  as_fn_append MODULE_BLOCK "MODULE__XXINTERPQUEUES_STATE=$py_cv_module__xxinterpqueues$as_nl"
+  if test "x$py_cv_module__xxinterpqueues" = xyes
+then :
+
+
+
+
 fi
 
 
@@ -30760,6 +30785,10 @@ if test -z "${MODULE__XXINTERPCHANNELS_TRUE}" && test -z "${MODULE__XXINTERPCHAN
   as_fn_error $? "conditional \"MODULE__XXINTERPCHANNELS\" was never defined.
 Usually this means the macro was only invoked conditionally." "$LINENO" 5
 fi
+if test -z "${MODULE__XXINTERPQUEUES_TRUE}" && test -z "${MODULE__XXINTERPQUEUES_FALSE}"; then
+  as_fn_error $? "conditional \"MODULE__XXINTERPQUEUES\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
 if test -z "${MODULE__ZONEINFO_TRUE}" && test -z "${MODULE__ZONEINFO_FALSE}"; then
   as_fn_error $? "conditional \"MODULE__ZONEINFO\" was never defined.
 Usually this means the macro was only invoked conditionally." "$LINENO" 5

configure.ac

@@ -7120,6 +7120,7 @@ AS_CASE([$ac_sys_system],
       [_tkinter],
       [_xxsubinterpreters],
       [_xxinterpchannels],
+      [_xxinterpqueues],
       [grp],
       [pwd],
       [resource],
@@ -7236,6 +7237,7 @@ PY_STDLIB_MOD_SIMPLE([_struct])
 PY_STDLIB_MOD_SIMPLE([_typing])
 PY_STDLIB_MOD_SIMPLE([_xxsubinterpreters])
 PY_STDLIB_MOD_SIMPLE([_xxinterpchannels])
+PY_STDLIB_MOD_SIMPLE([_xxinterpqueues])
 PY_STDLIB_MOD_SIMPLE([_zoneinfo])
 
 dnl multiprocessing modules