bpo-44353: Implement typing.NewType __call__ method in C (#27262)

Co-authored-by: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com>
Co-authored-by: Denis Laxalde <denis@laxalde.org>

Lib/test/test_typing.py

@@ -33,10 +33,15 @@
 import weakref
 import types
 
+from test.support import import_helper
 from test import mod_generics_cache
 from test import _typed_dict_helper
 
 
+py_typing = import_helper.import_fresh_module('typing', blocked=['_typing'])
+c_typing = import_helper.import_fresh_module('typing', fresh=['_typing'])
+
+
 class BaseTestCase(TestCase):
 
     def assertIsSubclass(self, cls, class_or_tuple, msg=None):
@@ -3673,18 +3678,36 @@ def foo(a: A) -> Optional[BaseException]:
         assert foo(None) is None
 
 
-class NewTypeTests(BaseTestCase):
+class TestModules(TestCase):
+    func_names = ['_idfunc']
+
+    def test_py_functions(self):
+        for fname in self.func_names:
+            self.assertEqual(getattr(py_typing, fname).__module__, 'typing')
+
+    @skipUnless(c_typing, 'requires _typing')
+    def test_c_functions(self):
+        for fname in self.func_names:
+            self.assertEqual(getattr(c_typing, fname).__module__, '_typing')
+
+
+class NewTypeTests:
+    def setUp(self):
+        sys.modules['typing'] = self.module
+
+    def tearDown(self):
+        sys.modules['typing'] = typing
 
     def test_basic(self):
-        UserId = NewType('UserId', int)
-        UserName = NewType('UserName', str)
+        UserId = self.module.NewType('UserId', int)
+        UserName = self.module.NewType('UserName', str)
         self.assertIsInstance(UserId(5), int)
         self.assertIsInstance(UserName('Joe'), str)
         self.assertEqual(UserId(5) + 1, 6)
 
     def test_errors(self):
-        UserId = NewType('UserId', int)
-        UserName = NewType('UserName', str)
+        UserId = self.module.NewType('UserId', int)
+        UserName = self.module.NewType('UserName', str)
         with self.assertRaises(TypeError):
             issubclass(UserId, int)
         with self.assertRaises(TypeError):
@@ -3692,29 +3715,38 @@ class D(UserName):
                 pass
 
     def test_or(self):
-        UserId = NewType('UserId', int)
-        UserName = NewType('UserName', str)
+        UserId = self.module.NewType('UserId', int)
+        UserName = self.module.NewType('UserName', str)
 
         for cls in (int, UserName):
             with self.subTest(cls=cls):
-                self.assertEqual(UserId | cls, Union[UserId, cls])
-                self.assertEqual(cls | UserId, Union[cls, UserId])
+                self.assertEqual(UserId | cls, self.module.Union[UserId, cls])
+                self.assertEqual(cls | UserId, self.module.Union[cls, UserId])
 
-                self.assertEqual(get_args(UserId | cls), (UserId, cls))
-                self.assertEqual(get_args(cls | UserId), (cls, UserId))
+                self.assertEqual(self.module.get_args(UserId | cls), (UserId, cls))
+                self.assertEqual(self.module.get_args(cls | UserId), (cls, UserId))
 
     def test_special_attrs(self):
-        UserId = NewType('UserId', int)
+        UserId = self.module.NewType('UserId', int)
 
         self.assertEqual(UserId.__name__, 'UserId')
         self.assertEqual(UserId.__qualname__, 'UserId')
         self.assertEqual(UserId.__module__, __name__)
 
     def test_repr(self):
-        UserId = NewType('UserId', int)
+        UserId = self.module.NewType('UserId', int)
 
         self.assertEqual(repr(UserId), f'{__name__}.UserId')
 
+class NewTypePythonTests(BaseTestCase, NewTypeTests):
+    module = py_typing
+
+
+@skipUnless(c_typing, 'requires _typing')
+class NewTypeCTests(BaseTestCase, NewTypeTests):
+    module = c_typing
+
+
 class NamedTupleTests(BaseTestCase):
     class NestedEmployee(NamedTuple):
         name: str

Lib/typing.py

@@ -31,6 +31,13 @@
 import warnings
 from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType, GenericAlias
 
+
+try:
+    from _typing import _idfunc
+except ImportError:
+    def _idfunc(_, x):
+        return x
+
 # Please keep __all__ alphabetized within each category.
 __all__ = [
     # Super-special typing primitives.
@@ -2375,6 +2382,8 @@ def name_by_id(user_id: UserId) -> str:
         num = UserId(5) + 1     # type: int
     """
 
+    __call__ = _idfunc
+
     def __init__(self, name, tp):
         self.__name__ = name
         self.__qualname__ = name
@@ -2384,9 +2393,6 @@ def __init__(self, name, tp):
     def __repr__(self):
         return f'{self.__module__}.{self.__qualname__}'
 
-    def __call__(self, x):
-        return x
-
     def __or__(self, other):
         return Union[self, other]
 

Misc/NEWS.d/next/Library/2021-07-20-18-34-16.bpo-44353.ATuYq4.rst

@@ -0,0 +1,2 @@
+Make ``NewType.__call__`` faster by implementing it in C.
+Patch provided by Yurii Karabas.

Modules/Setup

@@ -184,6 +184,7 @@ _symtable symtablemodule.c
 #_asyncio _asynciomodule.c  # Fast asyncio Future
 #_json -I$(srcdir)/Include/internal -DPy_BUILD_CORE_BUILTIN _json.c	# _json speedups
 #_statistics _statisticsmodule.c # statistics accelerator
+#_typing _typingmodule.c # typing accelerator
 
 #unicodedata unicodedata.c -DPy_BUILD_CORE_BUILTIN   # static Unicode character database
 

Modules/_typingmodule.c

@@ -0,0 +1,59 @@
+/* typing accelerator C extension: _typing module. */
+
+#include "Python.h"
+#include "clinic/_typingmodule.c.h"
+
+/*[clinic input]
+module _typing
+
+[clinic start generated code]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=1db35baf1c72942b]*/
+
+/* helper function to make typing.NewType.__call__ method faster */
+
+/*[clinic input]
+_typing._idfunc -> object
+
+    x: object
+    /
+
+[clinic start generated code]*/
+
+static PyObject *
+_typing__idfunc(PyObject *module, PyObject *x)
+/*[clinic end generated code: output=63c38be4a6ec5f2c input=49f17284b43de451]*/
+{
+    Py_INCREF(x);
+    return x;
+}
+
+
+static PyMethodDef typing_methods[] = {
+    _TYPING__IDFUNC_METHODDEF
+    {NULL, NULL, 0, NULL}
+};
+
+PyDoc_STRVAR(typing_doc,
+"Accelerators for the typing module.\n");
+
+static struct PyModuleDef_Slot _typingmodule_slots[] = {
+    {0, NULL}
+};
+
+static struct PyModuleDef typingmodule = {
+        PyModuleDef_HEAD_INIT,
+        "_typing",
+        typing_doc,
+        0,
+        typing_methods,
+        _typingmodule_slots,
+        NULL,
+        NULL,
+        NULL
+};
+
+PyMODINIT_FUNC
+PyInit__typing(void)
+{
+    return PyModuleDef_Init(&typingmodule);
+}

Modules/clinic/_typingmodule.c.h

@@ -0,0 +1,12 @@
+/*[clinic input]
+preserve
+[clinic start generated code]*/
+
+PyDoc_STRVAR(_typing__idfunc__doc__,
+"_idfunc($module, x, /)\n"
+"--\n"
+"\n");
+
+#define _TYPING__IDFUNC_METHODDEF    \
+    {"_idfunc", (PyCFunction)_typing__idfunc, METH_O, _typing__idfunc__doc__},
+/*[clinic end generated code: output=e7ea2a3cb7ab301a input=a9049054013a1b77]*/

PC/config.c

@@ -24,6 +24,7 @@ extern PyObject* PyInit__sha256(void);
 extern PyObject* PyInit__sha512(void);
 extern PyObject* PyInit__sha3(void);
 extern PyObject* PyInit__statistics(void);
+extern PyObject* PyInit__typing(void);
 extern PyObject* PyInit__blake2(void);
 extern PyObject* PyInit_time(void);
 extern PyObject* PyInit__thread(void);
@@ -104,6 +105,7 @@ struct _inittab _PyImport_Inittab[] = {
     {"_blake2", PyInit__blake2},
     {"time", PyInit_time},
     {"_thread", PyInit__thread},
+    {"_typing", PyInit__typing},
     {"_statistics", PyInit__statistics},
 #ifdef WIN32
     {"msvcrt", PyInit_msvcrt},

PCbuild/pythoncore.vcxproj

@@ -363,6 +363,7 @@
     <ClCompile Include="..\Modules\symtablemodule.c" />
     <ClCompile Include="..\Modules\_threadmodule.c" />
     <ClCompile Include="..\Modules\_tracemalloc.c" />
+    <ClCompile Include="..\Modules\_typingmodule.c" />
     <ClCompile Include="..\Modules\timemodule.c" />
     <ClCompile Include="..\Modules\xxsubtype.c" />
     <ClCompile Include="..\Modules\_xxsubinterpretersmodule.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -698,6 +698,9 @@
     <ClCompile Include="..\Modules\_statisticsmodule.c">
       <Filter>Modules</Filter>
     </ClCompile>
+    <ClCompile Include="..\Modules\_typingmodule.c">
+      <Filter>Modules</Filter>
+    </ClCompile>
     <ClCompile Include="..\Modules\_struct.c">
       <Filter>Modules</Filter>
     </ClCompile>

Python/stdlib_module_names.h

@@ -81,6 +81,7 @@ static const char* _Py_stdlib_module_names[] = {
 "_threading_local",
 "_tkinter",
 "_tracemalloc",
+"_typing",
 "_uuid",
 "_warnings",
 "_weakref",

setup.py

@@ -957,6 +957,8 @@ def detect_simple_extensions(self):
                            extra_compile_args=['-DPy_BUILD_CORE_MODULE']))
         # _statistics module
         self.add(Extension("_statistics", ["_statisticsmodule.c"]))
+        # _typing module
+        self.add(Extension("_typing", ["_typingmodule.c"]))
 
         # Modules with some UNIX dependencies -- on by default:
         # (If you have a really backward UNIX, select and socket may not be