diff --git a/Include/Python.h b/Include/Python.h index 969d8e6bea7..769ec49779c 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -120,6 +120,7 @@ #include "iterobject.h" #include "genobject.h" #include "descrobject.h" +#include "genericaliasobject.h" #include "warnings.h" #include "weakrefobject.h" #include "structseq.h" diff --git a/Include/genericaliasobject.h b/Include/genericaliasobject.h new file mode 100644 index 00000000000..cf002976b27 --- /dev/null +++ b/Include/genericaliasobject.h @@ -0,0 +1,14 @@ +// Implementation of PEP 585: support list[int] etc. +#ifndef Py_GENERICALIASOBJECT_H +#define Py_GENERICALIASOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +PyAPI_FUNC(PyObject *) Py_GenericAlias(PyObject *, PyObject *); +PyAPI_DATA(PyTypeObject) Py_GenericAliasType; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_GENERICALIASOBJECT_H */ diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 2b2ddba170e..36cd9930003 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -9,6 +9,8 @@ from abc import ABCMeta, abstractmethod import sys +GenericAlias = type(list[int]) + __all__ = ["Awaitable", "Coroutine", "AsyncIterable", "AsyncIterator", "AsyncGenerator", "Hashable", "Iterable", "Iterator", "Generator", "Reversible", @@ -110,6 +112,8 @@ def __subclasshook__(cls, C): return _check_methods(C, "__await__") return NotImplemented + __class_getitem__ = classmethod(GenericAlias) + class Coroutine(Awaitable): @@ -169,6 +173,8 @@ def __subclasshook__(cls, C): return _check_methods(C, "__aiter__") return NotImplemented + __class_getitem__ = classmethod(GenericAlias) + class AsyncIterator(AsyncIterable): @@ -255,6 +261,8 @@ def __subclasshook__(cls, C): return _check_methods(C, "__iter__") return NotImplemented + __class_getitem__ = classmethod(GenericAlias) + class Iterator(Iterable): @@ -274,6 +282,7 @@ def __subclasshook__(cls, C): return _check_methods(C, '__iter__', '__next__') return NotImplemented + Iterator.register(bytes_iterator) Iterator.register(bytearray_iterator) #Iterator.register(callable_iterator) @@ -353,6 +362,7 @@ def __subclasshook__(cls, C): 'send', 'throw', 'close') return NotImplemented + Generator.register(generator) @@ -385,6 +395,9 @@ def __subclasshook__(cls, C): return _check_methods(C, "__contains__") return NotImplemented + __class_getitem__ = classmethod(GenericAlias) + + class Collection(Sized, Iterable, Container): __slots__ = () @@ -395,6 +408,7 @@ def __subclasshook__(cls, C): return _check_methods(C, "__len__", "__iter__", "__contains__") return NotImplemented + class Callable(metaclass=ABCMeta): __slots__ = () @@ -409,6 +423,8 @@ def __subclasshook__(cls, C): return _check_methods(C, "__call__") return NotImplemented + __class_getitem__ = classmethod(GenericAlias) + ### SETS ### @@ -550,6 +566,7 @@ def _hash(self): h = 590923713 return h + Set.register(frozenset) @@ -632,6 +649,7 @@ def __isub__(self, it): self.discard(value) return self + MutableSet.register(set) @@ -688,6 +706,7 @@ def __eq__(self, other): __reversed__ = None + Mapping.register(mappingproxy) @@ -704,6 +723,8 @@ def __len__(self): def __repr__(self): return '{0.__class__.__name__}({0._mapping!r})'.format(self) + __class_getitem__ = classmethod(GenericAlias) + class KeysView(MappingView, Set): @@ -719,6 +740,7 @@ def __contains__(self, key): def __iter__(self): yield from self._mapping + KeysView.register(dict_keys) @@ -743,6 +765,7 @@ def __iter__(self): for key in self._mapping: yield (key, self._mapping[key]) + ItemsView.register(dict_items) @@ -761,6 +784,7 @@ def __iter__(self): for key in self._mapping: yield self._mapping[key] + ValuesView.register(dict_values) @@ -847,6 +871,7 @@ def setdefault(self, key, default=None): self[key] = default return default + MutableMapping.register(dict) @@ -914,6 +939,7 @@ def count(self, value): 'S.count(value) -> integer -- return number of occurrences of value' return sum(1 for v in self if v is value or v == value) + Sequence.register(tuple) Sequence.register(str) Sequence.register(range) @@ -1000,5 +1026,6 @@ def __iadd__(self, values): self.extend(values) return self + MutableSequence.register(list) MutableSequence.register(bytearray) # Multiply inheriting, see ByteString diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 69c272831a5..ff92d9f913f 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -4,7 +4,7 @@ import _collections_abc from collections import deque from functools import wraps -from types import MethodType +from types import MethodType, GenericAlias __all__ = ["asynccontextmanager", "contextmanager", "closing", "nullcontext", "AbstractContextManager", "AbstractAsyncContextManager", @@ -16,6 +16,8 @@ class AbstractContextManager(abc.ABC): """An abstract base class for context managers.""" + __class_getitem__ = classmethod(GenericAlias) + def __enter__(self): """Return `self` upon entering the runtime context.""" return self @@ -36,6 +38,8 @@ class AbstractAsyncContextManager(abc.ABC): """An abstract base class for asynchronous context managers.""" + __class_getitem__ = classmethod(GenericAlias) + async def __aenter__(self): """Return `self` upon entering the runtime context.""" return self diff --git a/Lib/os.py b/Lib/os.py index 8acd6f12c3e..b794159f86c 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -28,6 +28,8 @@ from _collections_abc import _check_methods +GenericAlias = type(list[int]) + _names = sys.builtin_module_names # Note: more names are added to __all__ later. @@ -1074,8 +1076,7 @@ def __subclasshook__(cls, subclass): return _check_methods(subclass, '__fspath__') return NotImplemented - def __class_getitem__(cls, type): - return cls + __class_getitem__ = classmethod(GenericAlias) if name == 'nt': diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 1eecceaed41..86fdf27f9b0 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -52,6 +52,7 @@ import warnings import contextlib from time import monotonic as _time +import types try: import pwd @@ -446,17 +447,7 @@ def __repr__(self): args.append('stderr={!r}'.format(self.stderr)) return "{}({})".format(type(self).__name__, ', '.join(args)) - def __class_getitem__(cls, type): - """Provide minimal support for using this class as generic - (for example in type annotations). - - See PEP 484 and PEP 560 for more details. For example, - `CompletedProcess[bytes]` is a valid expression at runtime - (type argument `bytes` indicates the type used for stdout). - Note, no type checking happens at runtime, but a static type - checker can be used. - """ - return cls + __class_getitem__ = classmethod(types.GenericAlias) def check_returncode(self): @@ -1000,16 +991,7 @@ def __repr__(self): obj_repr = obj_repr[:76] + "...>" return obj_repr - def __class_getitem__(cls, type): - """Provide minimal support for using this class as generic - (for example in type annotations). - - See PEP 484 and PEP 560 for more details. For example, `Popen[bytes]` - is a valid expression at runtime (type argument `bytes` indicates the - type used for stdout). Note, no type checking happens at runtime, but - a static type checker can be used. - """ - return cls + __class_getitem__ = classmethod(types.GenericAlias) @property def universal_newlines(self): diff --git a/Lib/tempfile.py b/Lib/tempfile.py index b27165bb0f2..a398345f108 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -44,6 +44,7 @@ import errno as _errno from random import Random as _Random import sys as _sys +import types as _types import weakref as _weakref import _thread _allocate_lock = _thread.allocate_lock @@ -643,17 +644,7 @@ def __init__(self, max_size=0, mode='w+b', buffering=-1, 'encoding': encoding, 'newline': newline, 'dir': dir, 'errors': errors} - def __class_getitem__(cls, type): - """Provide minimal support for using this class as generic - (for example in type annotations). - - See PEP 484 and PEP 560 for more details. For example, - `SpooledTemporaryFile[str]` is a valid expression at runtime (type - argument `str` indicates whether the file is open in bytes or text - mode). Note, no type checking happens at runtime, but a static type - checker can be used. - """ - return cls + __class_getitem__ = classmethod(_types.GenericAlias) def _check(self, file): if self._rolled: return diff --git a/Lib/test/test_descrtut.py b/Lib/test/test_descrtut.py index b84d6447850..8e25f58d7aa 100644 --- a/Lib/test/test_descrtut.py +++ b/Lib/test/test_descrtut.py @@ -167,6 +167,7 @@ def merge(self, other): >>> pprint.pprint(dir(list)) # like list.__dict__.keys(), but sorted ['__add__', '__class__', + '__class_getitem__', '__contains__', '__delattr__', '__delitem__', diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 9e88222e953..71426277d2d 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -665,7 +665,7 @@ def non_Python_modules(): r""" >>> import builtins >>> tests = doctest.DocTestFinder().find(builtins) - >>> 800 < len(tests) < 820 # approximate number of objects with docstrings + >>> 810 < len(tests) < 830 # approximate number of objects with docstrings True >>> real_tests = [t for t in tests if len(t.examples) > 0] >>> len(real_tests) # objects that actually have doctests diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py new file mode 100644 index 00000000000..9d8dbfe0420 --- /dev/null +++ b/Lib/test/test_genericalias.py @@ -0,0 +1,214 @@ +"""Tests for C-implemented GenericAlias.""" + +import unittest +import pickle +from collections import ( + defaultdict, deque, OrderedDict, Counter, UserDict, UserList +) +from collections.abc import * +from contextlib import AbstractContextManager, AbstractAsyncContextManager +from re import Pattern, Match +from types import GenericAlias, MappingProxyType +import typing + +from typing import TypeVar +T = TypeVar('T') + +class BaseTest(unittest.TestCase): + """Test basics.""" + + def test_subscriptable(self): + for t in (type, tuple, list, dict, set, frozenset, + defaultdict, deque, + OrderedDict, Counter, UserDict, UserList, + Pattern, Match, + AbstractContextManager, AbstractAsyncContextManager, + Awaitable, Coroutine, + AsyncIterable, AsyncIterator, + AsyncGenerator, Generator, + Iterable, Iterator, + Reversible, + Container, Collection, + Callable, + Set, MutableSet, + Mapping, MutableMapping, MappingView, + KeysView, ItemsView, ValuesView, + Sequence, MutableSequence, + MappingProxyType, + ): + tname = t.__name__ + with self.subTest(f"Testing {tname}"): + alias = t[int] + self.assertIs(alias.__origin__, t) + self.assertEqual(alias.__args__, (int,)) + self.assertEqual(alias.__parameters__, ()) + + def test_unsubscriptable(self): + for t in int, str, float, Sized, Hashable: + tname = t.__name__ + with self.subTest(f"Testing {tname}"): + with self.assertRaises(TypeError): + t[int] + + def test_instantiate(self): + for t in tuple, list, dict, set, frozenset, defaultdict, deque: + tname = t.__name__ + with self.subTest(f"Testing {tname}"): + alias = t[int] + self.assertEqual(alias(), t()) + if t is dict: + self.assertEqual(alias(iter([('a', 1), ('b', 2)])), dict(a=1, b=2)) + self.assertEqual(alias(a=1, b=2), dict(a=1, b=2)) + elif t is defaultdict: + def default(): + return 'value' + a = alias(default) + d = defaultdict(default) + self.assertEqual(a['test'], d['test']) + else: + self.assertEqual(alias(iter((1, 2, 3))), t((1, 2, 3))) + + def test_unbound_methods(self): + t = list[int] + a = t() + t.append(a, 'foo') + self.assertEqual(a, ['foo']) + x = t.__getitem__(a, 0) + self.assertEqual(x, 'foo') + self.assertEqual(t.__len__(a), 1) + + def test_subclassing(self): + class C(list[int]): + pass + self.assertEqual(C.__bases__, (list,)) + self.assertEqual(C.__class__, type) + + def test_class_methods(self): + t = dict[int, None] + self.assertEqual(dict.fromkeys(range(2)), {0: None, 1: None}) # This works + self.assertEqual(t.fromkeys(range(2)), {0: None, 1: None}) # Should be equivalent + + def test_no_chaining(self): + t = list[int] + with self.assertRaises(TypeError): + t[int] + + def test_generic_subclass(self): + class MyList(list): + pass + t = MyList[int] + self.assertIs(t.__origin__, MyList) + self.assertEqual(t.__args__, (int,)) + self.assertEqual(t.__parameters__, ()) + + def test_repr(self): + class MyList(list): + pass + self.assertEqual(repr(list[str]), 'list[str]') + self.assertEqual(repr(list[()]), 'list[()]') + self.assertEqual(repr(tuple[int, ...]), 'tuple[int, ...]') + self.assertTrue(repr(MyList[int]).endswith('.BaseTest.test_repr..MyList[int]')) + self.assertEqual(repr(list[str]()), '[]') # instances should keep their normal repr + + def test_exposed_type(self): + import types + a = types.GenericAlias(list, int) + self.assertEqual(str(a), 'list[int]') + self.assertIs(a.__origin__, list) + self.assertEqual(a.__args__, (int,)) + self.assertEqual(a.__parameters__, ()) + + def test_parameters(self): + from typing import TypeVar + T = TypeVar('T') + K = TypeVar('K') + V = TypeVar('V') + D0 = dict[str, int] + self.assertEqual(D0.__args__, (str, int)) + self.assertEqual(D0.__parameters__, ()) + D1a = dict[str, V] + self.assertEqual(D1a.__args__, (str, V)) + self.assertEqual(D1a.__parameters__, (V,)) + D1b = dict[K, int] + self.assertEqual(D1b.__args__, (K, int)) + self.assertEqual(D1b.__parameters__, (K,)) + D2a = dict[K, V] + self.assertEqual(D2a.__args__, (K, V)) + self.assertEqual(D2a.__parameters__, (K, V)) + D2b = dict[T, T] + self.assertEqual(D2b.__args__, (T, T)) + self.assertEqual(D2b.__parameters__, (T,)) + L0 = list[str] + self.assertEqual(L0.__args__, (str,)) + self.assertEqual(L0.__parameters__, ()) + L1 = list[T] + self.assertEqual(L1.__args__, (T,)) + self.assertEqual(L1.__parameters__, (T,)) + + def test_parameter_chaining(self): + from typing import TypeVar + T = TypeVar('T') + self.assertEqual(list[T][int], list[int]) + self.assertEqual(dict[str, T][int], dict[str, int]) + self.assertEqual(dict[T, int][str], dict[str, int]) + self.assertEqual(dict[T, T][int], dict[int, int]) + with self.assertRaises(TypeError): + list[int][int] + dict[T, int][str, int] + dict[str, T][str, int] + dict[T, T][str, int] + + def test_equality(self): + self.assertEqual(list[int], list[int]) + self.assertEqual(dict[str, int], dict[str, int]) + self.assertNotEqual(dict[str, int], dict[str, str]) + self.assertNotEqual(list, list[int]) + self.assertNotEqual(list[int], list) + + def test_isinstance(self): + self.assertTrue(isinstance([], list)) + with self.assertRaises(TypeError): + isinstance([], list[str]) + + def test_issubclass(self): + class L(list): ... + self.assertTrue(issubclass(L, list)) + with self.assertRaises(TypeError): + issubclass(L, list[str]) + + def test_type_generic(self): + t = type[int] + Test = t('Test', (), {}) + self.assertTrue(isinstance(Test, type)) + test = Test() + self.assertEqual(t(test), Test) + self.assertEqual(t(0), int) + + def test_type_subclass_generic(self): + class MyType(type): + pass + with self.assertRaises(TypeError): + MyType[int] + + def test_pickle(self): + alias = GenericAlias(list, T) + s = pickle.dumps(alias) + loaded = pickle.loads(s) + self.assertEqual(alias.__origin__, loaded.__origin__) + self.assertEqual(alias.__args__, loaded.__args__) + self.assertEqual(alias.__parameters__, loaded.__parameters__) + + def test_union(self): + a = typing.Union[list[int], list[str]] + self.assertEqual(a.__args__, (list[int], list[str])) + self.assertEqual(a.__parameters__, ()) + + def test_union_generic(self): + T = typing.TypeVar('T') + a = typing.Union[list[T], tuple[T, ...]] + self.assertEqual(a.__args__, (list[T], tuple[T, ...])) + self.assertEqual(a.__parameters__, (T,)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 142cfea364e..73dc064d5ff 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -25,6 +25,7 @@ import tempfile import threading import time +import types import unittest import uuid import warnings @@ -4191,7 +4192,7 @@ class A(os.PathLike): self.assertTrue(issubclass(FakePath, os.PathLike)) def test_pathlike_class_getitem(self): - self.assertIs(os.PathLike[bytes], os.PathLike) + self.assertIsInstance(os.PathLike[bytes], types.GenericAlias) class TimesTests(unittest.TestCase): diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 7cf31e1f092..aced87694cf 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -11,6 +11,7 @@ import tempfile import time import traceback +import types import selectors import sysconfig import select @@ -1435,8 +1436,8 @@ def test_file_not_found_with_bad_cwd(self): self.assertEqual(c.exception.filename, '/some/nonexistent/directory') def test_class_getitems(self): - self.assertIs(subprocess.Popen[bytes], subprocess.Popen) - self.assertIs(subprocess.CompletedProcess[str], subprocess.CompletedProcess) + self.assertIsInstance(subprocess.Popen[bytes], types.GenericAlias) + self.assertIsInstance(subprocess.CompletedProcess[str], types.GenericAlias) class RunFuncTestCase(BaseTestCase): def run_python(self, code, **kwargs): diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 524ab7c6d76..f4cba0fa8dc 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -10,6 +10,7 @@ import warnings import contextlib import stat +import types import weakref from unittest import mock @@ -1222,8 +1223,8 @@ def test_truncate_with_size_parameter(self): self.assertEqual(os.fstat(f.fileno()).st_size, 20) def test_class_getitem(self): - self.assertIs(tempfile.SpooledTemporaryFile[bytes], - tempfile.SpooledTemporaryFile) + self.assertIsInstance(tempfile.SpooledTemporaryFile[bytes], + types.GenericAlias) if tempfile.NamedTemporaryFile is not tempfile.TemporaryFile: diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 544c91bc36a..f42238762dd 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -622,6 +622,7 @@ def test_methods(self): self.assertEqual(attrs, { '__contains__', '__getitem__', + '__class_getitem__', '__ior__', '__iter__', '__len__', diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index dea09eb874d..95f865f8c34 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1775,10 +1775,11 @@ def __call__(self): self.assertEqual(T1[int, T].__origin__, T1) self.assertEqual(T2.__parameters__, (T,)) - with self.assertRaises(TypeError): - T1[int] - with self.assertRaises(TypeError): - T2[int, str] + # These don't work because of tuple.__class_item__ + ## with self.assertRaises(TypeError): + ## T1[int] + ## with self.assertRaises(TypeError): + ## T2[int, str] self.assertEqual(repr(C1[int]).split('.')[-1], 'C1[int]') self.assertEqual(C2.__parameters__, ()) @@ -1817,22 +1818,22 @@ def test_type_erasure_special(self): self.clear_caches() class MyTup(Tuple[T, T]): ... self.assertIs(MyTup[int]().__class__, MyTup) - self.assertIs(MyTup[int]().__orig_class__, MyTup[int]) + self.assertEqual(MyTup[int]().__orig_class__, MyTup[int]) class MyCall(Callable[..., T]): def __call__(self): return None self.assertIs(MyCall[T]().__class__, MyCall) - self.assertIs(MyCall[T]().__orig_class__, MyCall[T]) + self.assertEqual(MyCall[T]().__orig_class__, MyCall[T]) class MyDict(typing.Dict[T, T]): ... self.assertIs(MyDict[int]().__class__, MyDict) - self.assertIs(MyDict[int]().__orig_class__, MyDict[int]) + self.assertEqual(MyDict[int]().__orig_class__, MyDict[int]) class MyDef(typing.DefaultDict[str, T]): ... self.assertIs(MyDef[int]().__class__, MyDef) - self.assertIs(MyDef[int]().__orig_class__, MyDef[int]) + self.assertEqual(MyDef[int]().__orig_class__, MyDef[int]) # ChainMap was added in 3.3 if sys.version_info >= (3, 3): class MyChain(typing.ChainMap[str, T]): ... self.assertIs(MyChain[int]().__class__, MyChain) - self.assertIs(MyChain[int]().__orig_class__, MyChain[int]) + self.assertEqual(MyChain[int]().__orig_class__, MyChain[int]) def test_all_repr_eq_any(self): objs = (getattr(typing, el) for el in typing.__all__) diff --git a/Lib/types.py b/Lib/types.py index ea3c0b29d5d..ad2020ec69b 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -293,4 +293,7 @@ def wrapped(*args, **kwargs): return wrapped +GenericAlias = type(list[int]) + + __all__ = [n for n in globals() if n[:1] != '_'] diff --git a/Lib/typing.py b/Lib/typing.py index a72003a4a96..6cc3b0342ec 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -26,7 +26,7 @@ import re as stdlib_re # Avoid confusion with the re we export. import sys import types -from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType +from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType, GenericAlias # Please keep __all__ alphabetized within each category. __all__ = [ @@ -180,7 +180,8 @@ def _collect_type_vars(types): for t in types: if isinstance(t, TypeVar) and t not in tvars: tvars.append(t) - if isinstance(t, _GenericAlias) and not t._special: + if ((isinstance(t, _GenericAlias) and not t._special) + or isinstance(t, GenericAlias)): tvars.extend([t for t in t.__parameters__ if t not in tvars]) return tuple(tvars) @@ -947,7 +948,7 @@ class _TypingEllipsis: _SPECIAL_NAMES = ['__abstractmethods__', '__annotations__', '__dict__', '__doc__', '__init__', '__module__', '__new__', '__slots__', - '__subclasshook__', '__weakref__'] + '__subclasshook__', '__weakref__', '__class_getitem__'] # These special attributes will be not collected as protocol members. EXCLUDED_ATTRIBUTES = _TYPING_INTERNALS + _SPECIAL_NAMES + ['_MutableMapping__marker'] diff --git a/Makefile.pre.in b/Makefile.pre.in index 0e3998c0ba4..948f9851d0c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -393,6 +393,7 @@ OBJECT_OBJS= \ Objects/descrobject.o \ Objects/enumobject.o \ Objects/exceptions.o \ + Objects/genericaliasobject.o \ Objects/genobject.o \ Objects/fileobject.o \ Objects/floatobject.o \ diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-01-28-17-19-18.bpo-39481.rqSeGl.rst b/Misc/NEWS.d/next/Core and Builtins/2020-01-28-17-19-18.bpo-39481.rqSeGl.rst new file mode 100644 index 00000000000..9652a3fccd7 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-01-28-17-19-18.bpo-39481.rqSeGl.rst @@ -0,0 +1 @@ +Implement PEP 585. This supports list[int], tuple[str, ...] etc. \ No newline at end of file diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 11342a35179..181e3229da4 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1609,6 +1609,8 @@ static PyMethodDef deque_methods[] = { METH_FASTCALL, rotate_doc}, {"__sizeof__", (PyCFunction)deque_sizeof, METH_NOARGS, sizeof_doc}, + {"__class_getitem__", (PyCFunction)Py_GenericAlias, + METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ }; @@ -2074,6 +2076,8 @@ static PyMethodDef defdict_methods[] = { defdict_copy_doc}, {"__reduce__", (PyCFunction)defdict_reduce, METH_NOARGS, reduce_doc}, + {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("See PEP 585")}, {NULL} }; diff --git a/Modules/_sre.c b/Modules/_sre.c index 15950c6b60c..52ed42099ba 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -2568,6 +2568,8 @@ static PyMethodDef pattern_methods[] = { _SRE_SRE_PATTERN_SCANNER_METHODDEF _SRE_SRE_PATTERN___COPY___METHODDEF _SRE_SRE_PATTERN___DEEPCOPY___METHODDEF + {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("See PEP 585")}, {NULL, NULL} }; @@ -2638,6 +2640,8 @@ static PyMethodDef match_methods[] = { _SRE_SRE_MATCH_EXPAND_METHODDEF _SRE_SRE_MATCH___COPY___METHODDEF _SRE_SRE_MATCH___DEEPCOPY___METHODDEF + {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("See PEP 585")}, {NULL, NULL} }; diff --git a/Objects/abstract.c b/Objects/abstract.c index 02e4ad70718..e975edd7c5b 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -176,6 +176,12 @@ PyObject_GetItem(PyObject *o, PyObject *key) if (PyType_Check(o)) { PyObject *meth, *result; _Py_IDENTIFIER(__class_getitem__); + + // Special case type[int], but disallow other types so str[int] fails + if ((PyTypeObject*)o == &PyType_Type) { + return Py_GenericAlias(o, key); + } + if (_PyObject_LookupAttrId(o, &PyId___class_getitem__, &meth) < 0) { return NULL; } diff --git a/Objects/descrobject.c b/Objects/descrobject.c index cb7572b0d5d..5fab222b82c 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1092,6 +1092,8 @@ static PyMethodDef mappingproxy_methods[] = { PyDoc_STR("D.items() -> list of D's (key, value) pairs, as 2-tuples")}, {"copy", (PyCFunction)mappingproxy_copy, METH_NOARGS, PyDoc_STR("D.copy() -> a shallow copy of D")}, + {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("See PEP 585")}, {0} }; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index e5f7005d49f..60660adf897 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -3252,6 +3252,7 @@ static PyMethodDef mapp_methods[] = { {"copy", (PyCFunction)dict_copy, METH_NOARGS, copy__doc__}, DICT___REVERSED___METHODDEF + {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c new file mode 100644 index 00000000000..49f537ef968 --- /dev/null +++ b/Objects/genericaliasobject.c @@ -0,0 +1,507 @@ +// types.GenericAlias -- used to represent e.g. list[int]. + +#include "Python.h" +#include "pycore_object.h" +#include "structmember.h" + +typedef struct { + PyObject_HEAD + PyObject *origin; + PyObject *args; + PyObject *parameters; +} gaobject; + +static void +ga_dealloc(PyObject *self) +{ + gaobject *alias = (gaobject *)self; + + _PyObject_GC_UNTRACK(self); + Py_XDECREF(alias->origin); + Py_XDECREF(alias->args); + Py_XDECREF(alias->parameters); + self->ob_type->tp_free(self); +} + +static int +ga_traverse(PyObject *self, visitproc visit, void *arg) +{ + gaobject *alias = (gaobject *)self; + Py_VISIT(alias->origin); + Py_VISIT(alias->args); + Py_VISIT(alias->parameters); + return 0; +} + +static int +ga_repr_item(_PyUnicodeWriter *writer, PyObject *p) +{ + _Py_IDENTIFIER(__module__); + _Py_IDENTIFIER(__qualname__); + _Py_IDENTIFIER(__origin__); + _Py_IDENTIFIER(__args__); + PyObject *qualname = NULL; + PyObject *module = NULL; + PyObject *r = NULL; + PyObject *tmp; + int err; + + if (p == Py_Ellipsis) { + // The Ellipsis object + r = PyUnicode_FromString("..."); + goto done; + } + + if (_PyObject_LookupAttrId(p, &PyId___origin__, &tmp) < 0) { + goto done; + } + if (tmp != NULL) { + Py_DECREF(tmp); + if (_PyObject_LookupAttrId(p, &PyId___args__, &tmp) < 0) { + goto done; + } + if (tmp != NULL) { + Py_DECREF(tmp); + // It looks like a GenericAlias + goto use_repr; + } + } + + if (_PyObject_LookupAttrId(p, &PyId___qualname__, &qualname) < 0) { + goto done; + } + if (qualname == NULL) { + goto use_repr; + } + if (_PyObject_LookupAttrId(p, &PyId___module__, &module) < 0) { + goto done; + } + if (module == NULL || module == Py_None) { + goto use_repr; + } + + // Looks like a class + if (PyUnicode_Check(module) && + _PyUnicode_EqualToASCIIString(module, "builtins")) + { + // builtins don't need a module name + r = PyObject_Str(qualname); + goto done; + } + else { + r = PyUnicode_FromFormat("%S.%S", module, qualname); + goto done; + } + +use_repr: + r = PyObject_Repr(p); + +done: + Py_XDECREF(qualname); + Py_XDECREF(module); + if (r == NULL) { + // error if any of the above PyObject_Repr/PyUnicode_From* fail + err = -1; + } + else { + err = _PyUnicodeWriter_WriteStr(writer, r); + Py_DECREF(r); + } + return err; +} + +static PyObject * +ga_repr(PyObject *self) +{ + gaobject *alias = (gaobject *)self; + Py_ssize_t len = PyTuple_GET_SIZE(alias->args); + + _PyUnicodeWriter writer; + _PyUnicodeWriter_Init(&writer); + + if (ga_repr_item(&writer, alias->origin) < 0) { + goto error; + } + if (_PyUnicodeWriter_WriteASCIIString(&writer, "[", 1) < 0) { + goto error; + } + for (Py_ssize_t i = 0; i < len; i++) { + if (i > 0) { + if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) { + goto error; + } + } + PyObject *p = PyTuple_GET_ITEM(alias->args, i); + if (ga_repr_item(&writer, p) < 0) { + goto error; + } + } + if (len == 0) { + // for something like tuple[()] we should print a "()" + if (_PyUnicodeWriter_WriteASCIIString(&writer, "()", 2) < 0) { + goto error; + } + } + if (_PyUnicodeWriter_WriteASCIIString(&writer, "]", 1) < 0) { + goto error; + } + return _PyUnicodeWriter_Finish(&writer); +error: + _PyUnicodeWriter_Dealloc(&writer); + return NULL; +} + +// isinstance(obj, TypeVar) without importing typing.py. +// Returns -1 for errors. +static int +is_typevar(PyObject *obj) +{ + PyTypeObject *type = Py_TYPE(obj); + if (strcmp(type->tp_name, "TypeVar") != 0) { + return 0; + } + PyObject *module = PyObject_GetAttrString((PyObject *)type, "__module__"); + if (module == NULL) { + return -1; + } + int res = PyUnicode_Check(module) + && _PyUnicode_EqualToASCIIString(module, "typing"); + Py_DECREF(module); + return res; +} + +// Index of item in self[:len], or -1 if not found (self is a tuple) +static Py_ssize_t +tuple_index(PyObject *self, Py_ssize_t len, PyObject *item) +{ + for (Py_ssize_t i = 0; i < len; i++) { + if (PyTuple_GET_ITEM(self, i) == item) { + return i; + } + } + return -1; +} + +// tuple(t for t in args if isinstance(t, TypeVar)) +static PyObject * +make_parameters(PyObject *args) +{ + Py_ssize_t len = PyTuple_GET_SIZE(args); + PyObject *parameters = PyTuple_New(len); + if (parameters == NULL) + return NULL; + Py_ssize_t iparam = 0; + for (Py_ssize_t iarg = 0; iarg < len; iarg++) { + PyObject *t = PyTuple_GET_ITEM(args, iarg); + int typevar = is_typevar(t); + if (typevar < 0) { + Py_XDECREF(parameters); + return NULL; + } + if (typevar) { + if (tuple_index(parameters, iparam, t) < 0) { + Py_INCREF(t); + PyTuple_SET_ITEM(parameters, iparam, t); + iparam++; + } + } + } + if (iparam < len) { + if (_PyTuple_Resize(¶meters, iparam) < 0) { + Py_XDECREF(parameters); + return NULL; + } + } + return parameters; +} + +static PyObject * +ga_getitem(PyObject *self, PyObject *item) +{ + gaobject *alias = (gaobject *)self; + // do a lookup for __parameters__ so it gets populated (if not already) + if (alias->parameters == NULL) { + alias->parameters = make_parameters(alias->args); + if (alias->parameters == NULL) { + return NULL; + } + } + Py_ssize_t nparams = PyTuple_GET_SIZE(alias->parameters); + if (nparams == 0) { + return PyErr_Format(PyExc_TypeError, + "There are no type variables left in %R", + self); + } + int is_tuple = PyTuple_Check(item); + Py_ssize_t nitem = is_tuple ? PyTuple_GET_SIZE(item) : 1; + if (nitem != nparams) { + return PyErr_Format(PyExc_TypeError, + "Too %s arguments for %R", + nitem > nparams ? "many" : "few", + self); + } + Py_ssize_t nargs = PyTuple_GET_SIZE(alias->args); + PyObject *newargs = PyTuple_New(nargs); + if (newargs == NULL) + return NULL; + for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { + PyObject *arg = PyTuple_GET_ITEM(alias->args, iarg); + int typevar = is_typevar(arg); + if (typevar < 0) { + Py_DECREF(newargs); + return NULL; + } + if (typevar) { + Py_ssize_t iparam = tuple_index(alias->parameters, nparams, arg); + assert(iparam >= 0); + if (is_tuple) { + arg = PyTuple_GET_ITEM(item, iparam); + } + else { + assert(iparam == 0); + arg = item; + } + } + Py_INCREF(arg); + PyTuple_SET_ITEM(newargs, iarg, arg); + } + PyObject *res = Py_GenericAlias(alias->origin, newargs); + Py_DECREF(newargs); + return res; +} + +static PyMappingMethods ga_as_mapping = { + .mp_subscript = ga_getitem, +}; + +static Py_hash_t +ga_hash(PyObject *self) +{ + gaobject *alias = (gaobject *)self; + // TODO: Hash in the hash for the origin + Py_hash_t h0 = PyObject_Hash(alias->origin); + if (h0 == -1) { + return -1; + } + Py_hash_t h1 = PyObject_Hash(alias->args); + if (h1 == -1) { + return -1; + } + return h0 ^ h1; +} + +static PyObject * +ga_call(PyObject *self, PyObject *args, PyObject *kwds) +{ + gaobject *alias = (gaobject *)self; + PyObject *obj = PyObject_Call(alias->origin, args, kwds); + if (obj != NULL) { + if (PyObject_SetAttrString(obj, "__orig_class__", self) < 0) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError) && + !PyErr_ExceptionMatches(PyExc_TypeError)) + { + Py_DECREF(obj); + return NULL; + } + PyErr_Clear(); + } + } + return obj; +} + +static const char* const attr_exceptions[] = { + "__origin__", + "__args__", + "__parameters__", + "__mro_entries__", + "__reduce_ex__", // needed so we don't look up object.__reduce_ex__ + "__reduce__", + NULL, +}; + +static PyObject * +ga_getattro(PyObject *self, PyObject *name) +{ + gaobject *alias = (gaobject *)self; + if (PyUnicode_Check(name)) { + for (const char * const *p = attr_exceptions; ; p++) { + if (*p == NULL) { + return PyObject_GetAttr(alias->origin, name); + } + if (_PyUnicode_EqualToASCIIString(name, *p)) { + break; + } + } + } + return PyObject_GenericGetAttr(self, name); +} + +static PyObject * +ga_richcompare(PyObject *a, PyObject *b, int op) +{ + if (Py_TYPE(a) != &Py_GenericAliasType || + Py_TYPE(b) != &Py_GenericAliasType || + (op != Py_EQ && op != Py_NE)) + { + Py_RETURN_NOTIMPLEMENTED; + } + + if (op == Py_NE) { + PyObject *eq = ga_richcompare(a, b, Py_EQ); + if (eq == NULL) + return NULL; + Py_DECREF(eq); + if (eq == Py_True) { + Py_RETURN_FALSE; + } + else { + Py_RETURN_TRUE; + } + } + + gaobject *aa = (gaobject *)a; + gaobject *bb = (gaobject *)b; + int eq = PyObject_RichCompareBool(aa->origin, bb->origin, Py_EQ); + if (eq < 0) { + return NULL; + } + if (!eq) { + Py_RETURN_FALSE; + } + return PyObject_RichCompare(aa->args, bb->args, Py_EQ); +} + +static PyObject * +ga_mro_entries(PyObject *self, PyObject *args) +{ + gaobject *alias = (gaobject *)self; + return PyTuple_Pack(1, alias->origin); +} + +static PyObject * +ga_instancecheck(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyErr_SetString(PyExc_TypeError, + "isinstance() argument 2 cannot be a parameterized generic"); + return NULL; +} + +static PyObject * +ga_subclasscheck(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyErr_SetString(PyExc_TypeError, + "issubclass() argument 2 cannot be a parameterized generic"); + return NULL; +} + +static PyObject * +ga_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + gaobject *alias = (gaobject *)self; + return Py_BuildValue("O(OO)", Py_TYPE(alias), + alias->origin, alias->args); +} + +static PyMethodDef ga_methods[] = { + {"__mro_entries__", ga_mro_entries, METH_O}, + {"__instancecheck__", ga_instancecheck, METH_O}, + {"__subclasscheck__", ga_subclasscheck, METH_O}, + {"__reduce__", ga_reduce, METH_NOARGS}, + {0} +}; + +static PyMemberDef ga_members[] = { + {"__origin__", T_OBJECT, offsetof(gaobject, origin), READONLY}, + {"__args__", T_OBJECT, offsetof(gaobject, args), READONLY}, + {0} +}; + +static PyObject * +ga_parameters(PyObject *self, void *unused) +{ + gaobject *alias = (gaobject *)self; + if (alias->parameters == NULL) { + alias->parameters = make_parameters(alias->args); + if (alias->parameters == NULL) { + return NULL; + } + } + Py_INCREF(alias->parameters); + return alias->parameters; +} + +static PyGetSetDef ga_properties[] = { + {"__parameters__", ga_parameters, (setter)NULL, "Type variables in the GenericAlias.", NULL}, + {0} +}; + +static PyObject * +ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + if (kwds != NULL && PyDict_GET_SIZE(kwds) != 0) { + PyErr_SetString(PyExc_TypeError, "GenericAlias does not support keyword arguments"); + return NULL; + } + if (PyTuple_GET_SIZE(args) != 2) { + PyErr_SetString(PyExc_TypeError, "GenericAlias expects 2 positional arguments"); + return NULL; + } + PyObject *origin = PyTuple_GET_ITEM(args, 0); + PyObject *arguments = PyTuple_GET_ITEM(args, 1); + return Py_GenericAlias(origin, arguments); +} + +// TODO: +// - argument clinic? +// - __doc__? +// - cache? +PyTypeObject Py_GenericAliasType = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "types.GenericAlias", + .tp_doc = "Represent a PEP 585 generic type\n" + "\n" + "E.g. for t = list[int], t.origin is list and t.args is (int,).", + .tp_basicsize = sizeof(gaobject), + .tp_dealloc = ga_dealloc, + .tp_repr = ga_repr, + .tp_as_mapping = &ga_as_mapping, + .tp_hash = ga_hash, + .tp_call = ga_call, + .tp_getattro = ga_getattro, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_traverse = ga_traverse, + .tp_richcompare = ga_richcompare, + .tp_methods = ga_methods, + .tp_members = ga_members, + .tp_alloc = PyType_GenericAlloc, + .tp_new = ga_new, + .tp_free = PyObject_GC_Del, + .tp_getset = ga_properties, +}; + +PyObject * +Py_GenericAlias(PyObject *origin, PyObject *args) +{ + if (!PyTuple_Check(args)) { + args = PyTuple_Pack(1, args); + if (args == NULL) { + return NULL; + } + } + else { + Py_INCREF(args); + } + + gaobject *alias = PyObject_GC_New(gaobject, &Py_GenericAliasType); + if (alias == NULL) { + Py_DECREF(args); + return NULL; + } + + Py_INCREF(origin); + alias->origin = origin; + alias->args = args; + alias->parameters = NULL; + _PyObject_GC_TRACK(alias); + return (PyObject *)alias; +} diff --git a/Objects/listobject.c b/Objects/listobject.c index 91687bcd229..48063b285f6 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2780,6 +2780,7 @@ static PyMethodDef list_methods[] = { LIST_COUNT_METHODDEF LIST_REVERSE_METHODDEF LIST_SORT_METHODDEF + {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/setobject.c b/Objects/setobject.c index 41b9ecd2f17..232ba6d97a0 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2132,6 +2132,7 @@ static PyMethodDef set_methods[] = { union_doc}, {"update", (PyCFunction)set_update, METH_VARARGS, update_doc}, + {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ }; @@ -2244,6 +2245,7 @@ static PyMethodDef frozenset_methods[] = { symmetric_difference_doc}, {"union", (PyCFunction)set_union, METH_VARARGS, union_doc}, + {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 8a8c6ae74f8..68163d8c48c 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -832,6 +832,7 @@ static PyMethodDef tuple_methods[] = { TUPLE___GETNEWARGS___METHODDEF TUPLE_INDEX_METHODDEF TUPLE_COUNT_METHODDEF + {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ }; diff --git a/PC/python3.def b/PC/python3.def index c7aed8789cf..083384e30f6 100644 --- a/PC/python3.def +++ b/PC/python3.def @@ -734,6 +734,8 @@ EXPORTS Py_FileSystemDefaultEncoding=python39.Py_FileSystemDefaultEncoding DATA Py_Finalize=python39.Py_Finalize Py_FinalizeEx=python39.Py_FinalizeEx + Py_GenericAlias=python39.Py_GenericAlias + Py_GenericAliasType=python39.Py_GenericAliasType Py_GetBuildInfo=python39.Py_GetBuildInfo Py_GetCompiler=python39.Py_GetCompiler Py_GetCopyright=python39.Py_GetCopyright diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 32778060e5f..22f1b95d0a7 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -383,6 +383,7 @@ +