From 884eba3c76916889fd6bff3b37b8552bfb4f9566 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 6 Apr 2022 20:00:14 +0300 Subject: [PATCH] bpo-26579: Add object.__getstate__(). (GH-2821) Copying and pickling instances of subclasses of builtin types bytearray, set, frozenset, collections.OrderedDict, collections.deque, weakref.WeakSet, and datetime.tzinfo now copies and pickles instance attributes implemented as slots. --- Doc/library/pickle.rst | 35 ++- Doc/whatsnew/3.11.rst | 9 + Include/object.h | 5 + Lib/_weakrefset.py | 3 +- Lib/collections/__init__.py | 20 +- Lib/copyreg.py | 4 + Lib/datetime.py | 10 +- Lib/email/headerregistry.py | 2 +- Lib/test/datetimetester.py | 6 +- Lib/test/pickletester.py | 4 +- Lib/test/test_bytes.py | 20 +- Lib/test/test_deque.py | 53 ++-- Lib/test/test_descrtut.py | 1 + Lib/test/test_ordered_dict.py | 39 ++- Lib/test/test_set.py | 24 +- Lib/test/test_weakset.py | 31 +++ Lib/test/test_xml_etree.py | 3 +- .../2017-07-23-11-28-45.bpo-26579.lpCY8R.rst | 7 + Modules/_collectionsmodule.c | 15 +- Modules/_datetimemodule.c | 30 +-- Objects/bytearrayobject.c | 29 +-- Objects/clinic/typeobject.c.h | 20 +- Objects/odictobject.c | 20 +- Objects/setobject.c | 14 +- Objects/typeobject.c | 240 ++++++++++-------- 25 files changed, 389 insertions(+), 255 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-07-23-11-28-45.bpo-26579.lpCY8R.rst diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index be48561ed10..a8ad5d437aa 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -509,9 +509,8 @@ The following types can be pickled: * classes that are defined at the top level of a module -* instances of such classes whose :attr:`~object.__dict__` or the result of - calling :meth:`__getstate__` is picklable (see section :ref:`pickle-inst` for - details). +* instances of such classes whose the result of calling :meth:`__getstate__` + is picklable (see section :ref:`pickle-inst` for details). Attempts to pickle unpicklable objects will raise the :exc:`PicklingError` exception; when this happens, an unspecified number of bytes may have already @@ -611,11 +610,31 @@ methods: .. method:: object.__getstate__() - Classes can further influence how their instances are pickled; if the class - defines the method :meth:`__getstate__`, it is called and the returned object - is pickled as the contents for the instance, instead of the contents of the - instance's dictionary. If the :meth:`__getstate__` method is absent, the - instance's :attr:`~object.__dict__` is pickled as usual. + Classes can further influence how their instances are pickled by overriding + the method :meth:`__getstate__`. It is called and the returned object + is pickled as the contents for the instance, instead of a default state. + There are several cases: + + * For a class that has no instance :attr:`~object.__dict__` and no + :attr:`~object.__slots__`, the default state is ``None``. + + * For a class that has an instance :attr:`~object.__dict__` and no + :attr:`~object.__slots__`, the default state is ``self.__dict__``. + + * For a class that has an instance :attr:`~object.__dict__` and + :attr:`~object.__slots__`, the default state is a tuple consisting of two + dictionaries: ``self.__dict__``, and a dictionary mapping slot + names to slot values. Only slots that have a value are + included in the latter. + + * For a class that has :attr:`~object.__slots__` and no instance + :attr:`~object.__dict__`, the default state is a tuple whose first item + is ``None`` and whose second item is a dictionary mapping slot names + to slot values described in the previous bullet. + + .. versionchanged:: 3.11 + Added the default implementation of the ``__getstate__()`` method in the + :class:`object` class. .. method:: object.__setstate__(state) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 96902cf61b4..4c9b32d9a94 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -187,6 +187,15 @@ Other Language Changes protocols correspondingly. (Contributed by Serhiy Storchaka in :issue:`12022`.) +* Added :meth:`object.__getstate__` which provides the default + implementation of the ``__getstate__()`` method. :mod:`Copying ` + and :mod:`pickling ` instances of subclasses of builtin types + :class:`bytearray`, :class:`set`, :class:`frozenset`, + :class:`collections.OrderedDict`, :class:`collections.deque`, + :class:`weakref.WeakSet`, and :class:`datetime.tzinfo` now copies and + pickles instance attributes implemented as :term:`slots <__slots__>`. + (Contributed by Serhiy Storchaka in :issue:`26579`.) + Other CPython Implementation Changes ==================================== diff --git a/Include/object.h b/Include/object.h index 317515d2c81..0b4b55ea1de 100644 --- a/Include/object.h +++ b/Include/object.h @@ -299,6 +299,11 @@ PyAPI_FUNC(void) PyObject_ClearWeakRefs(PyObject *); */ PyAPI_FUNC(PyObject *) PyObject_Dir(PyObject *); +/* Pickle support. */ +#ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject *) _PyObject_GetState(PyObject *); +#endif + /* Helpers for printing recursive container types */ PyAPI_FUNC(int) Py_ReprEnter(PyObject *); diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py index 2a27684324d..489eec714e0 100644 --- a/Lib/_weakrefset.py +++ b/Lib/_weakrefset.py @@ -80,8 +80,7 @@ def __contains__(self, item): return wr in self.data def __reduce__(self): - return (self.__class__, (list(self),), - getattr(self, '__dict__', None)) + return self.__class__, (list(self),), self.__getstate__() def add(self, item): if self._pending_removals: diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 264435cb41b..7af8dcd526d 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -271,10 +271,22 @@ def __repr__(self): def __reduce__(self): 'Return state information for pickling' - inst_dict = vars(self).copy() - for k in vars(OrderedDict()): - inst_dict.pop(k, None) - return self.__class__, (), inst_dict or None, None, iter(self.items()) + state = self.__getstate__() + if state: + if isinstance(state, tuple): + state, slots = state + else: + slots = {} + state = state.copy() + slots = slots.copy() + for k in vars(OrderedDict()): + state.pop(k, None) + slots.pop(k, None) + if slots: + state = state, slots + else: + state = state or None + return self.__class__, (), state, None, iter(self.items()) def copy(self): 'od.copy() -> a shallow copy of od' diff --git a/Lib/copyreg.py b/Lib/copyreg.py index 356db6f083e..c8a52a2dc63 100644 --- a/Lib/copyreg.py +++ b/Lib/copyreg.py @@ -89,6 +89,10 @@ def _reduce_ex(self, proto): except AttributeError: dict = None else: + if (type(self).__getstate__ is object.__getstate__ and + getattr(self, "__slots__", None)): + raise TypeError("a class that defines __slots__ without " + "defining __getstate__ cannot be pickled") dict = getstate() if dict: return _reconstructor, args, dict diff --git a/Lib/datetime.py b/Lib/datetime.py index 6bf37ccfab7..260b1de3887 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1169,15 +1169,7 @@ def __reduce__(self): args = getinitargs() else: args = () - getstate = getattr(self, "__getstate__", None) - if getstate: - state = getstate() - else: - state = getattr(self, "__dict__", None) or None - if state is None: - return (self.__class__, args) - else: - return (self.__class__, args, state) + return (self.__class__, args, self.__getstate__()) class IsoCalendarDate(tuple): diff --git a/Lib/email/headerregistry.py b/Lib/email/headerregistry.py index b590d69e8b7..543141dc427 100644 --- a/Lib/email/headerregistry.py +++ b/Lib/email/headerregistry.py @@ -218,7 +218,7 @@ def __reduce__(self): self.__class__.__bases__, str(self), ), - self.__dict__) + self.__getstate__()) @classmethod def _reconstruct(cls, value): diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index efea40d3f4f..335cded3b5f 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -139,8 +139,8 @@ class PicklableFixedOffset(FixedOffset): def __init__(self, offset=None, name=None, dstoffset=None): FixedOffset.__init__(self, offset, name, dstoffset) - def __getstate__(self): - return self.__dict__ +class PicklableFixedOffsetWithSlots(PicklableFixedOffset): + __slots__ = '_FixedOffset__offset', '_FixedOffset__name', 'spam' class _TZInfo(tzinfo): def utcoffset(self, datetime_module): @@ -202,6 +202,7 @@ def test_pickling_subclass(self): offset = timedelta(minutes=-300) for otype, args in [ (PicklableFixedOffset, (offset, 'cookie')), + (PicklableFixedOffsetWithSlots, (offset, 'cookie')), (timezone, (offset,)), (timezone, (offset, "EST"))]: orig = otype(*args) @@ -217,6 +218,7 @@ def test_pickling_subclass(self): self.assertIs(type(derived), otype) self.assertEqual(derived.utcoffset(None), offset) self.assertEqual(derived.tzname(None), oname) + self.assertFalse(hasattr(derived, 'spam')) def test_issue23600(self): DSTDIFF = DSTOFFSET = timedelta(hours=1) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index f13d42f6648..63fa7604fbf 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -2382,9 +2382,11 @@ def test_reduce_calls_base(self): def test_bad_getattr(self): # Issue #3514: crash when there is an infinite loop in __getattr__ x = BadGetattr() - for proto in protocols: + for proto in range(2): with support.infinite_recursion(): self.assertRaises(RuntimeError, self.dumps, x, proto) + for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): + s = self.dumps(x, proto) def test_reduce_bad_iterator(self): # Issue4176: crash when 4th and 5th items of __reduce__() diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index fd8e1d4bdc4..b457ff6ca84 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1940,28 +1940,30 @@ def test_join(self): def test_pickle(self): a = self.type2test(b"abcd") a.x = 10 - a.y = self.type2test(b"efgh") + a.z = self.type2test(b"efgh") for proto in range(pickle.HIGHEST_PROTOCOL + 1): b = pickle.loads(pickle.dumps(a, proto)) self.assertNotEqual(id(a), id(b)) self.assertEqual(a, b) self.assertEqual(a.x, b.x) - self.assertEqual(a.y, b.y) + self.assertEqual(a.z, b.z) self.assertEqual(type(a), type(b)) - self.assertEqual(type(a.y), type(b.y)) + self.assertEqual(type(a.z), type(b.z)) + self.assertFalse(hasattr(b, 'y')) def test_copy(self): a = self.type2test(b"abcd") a.x = 10 - a.y = self.type2test(b"efgh") + a.z = self.type2test(b"efgh") for copy_method in (copy.copy, copy.deepcopy): b = copy_method(a) self.assertNotEqual(id(a), id(b)) self.assertEqual(a, b) self.assertEqual(a.x, b.x) - self.assertEqual(a.y, b.y) + self.assertEqual(a.z, b.z) self.assertEqual(type(a), type(b)) - self.assertEqual(type(a.y), type(b.y)) + self.assertEqual(type(a.z), type(b.z)) + self.assertFalse(hasattr(b, 'y')) def test_fromhex(self): b = self.type2test.fromhex('1a2B30') @@ -1994,6 +1996,9 @@ def __init__(me, *args, **kwargs): class ByteArraySubclass(bytearray): pass +class ByteArraySubclassWithSlots(bytearray): + __slots__ = ('x', 'y', '__dict__') + class BytesSubclass(bytes): pass @@ -2014,6 +2019,9 @@ def __init__(me, newarg=1, *args, **kwargs): x = subclass(newarg=4, source=b"abcd") self.assertEqual(x, b"abcd") +class ByteArraySubclassWithSlotsTest(SubclassTest, unittest.TestCase): + basetype = bytearray + type2test = ByteArraySubclassWithSlots class BytesSubclassTest(SubclassTest, unittest.TestCase): basetype = bytes diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index 0be3feca010..ae1dfacd726 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -781,6 +781,9 @@ def test_runtime_error_on_empty_deque(self): class Deque(deque): pass +class DequeWithSlots(deque): + __slots__ = ('x', 'y', '__dict__') + class DequeWithBadIter(deque): def __iter__(self): raise TypeError @@ -810,40 +813,28 @@ def test_basics(self): self.assertEqual(len(d), 0) def test_copy_pickle(self): + for cls in Deque, DequeWithSlots: + for d in cls('abc'), cls('abcde', maxlen=4): + d.x = ['x'] + d.z = ['z'] - d = Deque('abc') + e = d.__copy__() + self.assertEqual(type(d), type(e)) + self.assertEqual(list(d), list(e)) - e = d.__copy__() - self.assertEqual(type(d), type(e)) - self.assertEqual(list(d), list(e)) + e = cls(d) + self.assertEqual(type(d), type(e)) + self.assertEqual(list(d), list(e)) - e = Deque(d) - self.assertEqual(type(d), type(e)) - self.assertEqual(list(d), list(e)) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - s = pickle.dumps(d, proto) - e = pickle.loads(s) - self.assertNotEqual(id(d), id(e)) - self.assertEqual(type(d), type(e)) - self.assertEqual(list(d), list(e)) - - d = Deque('abcde', maxlen=4) - - e = d.__copy__() - self.assertEqual(type(d), type(e)) - self.assertEqual(list(d), list(e)) - - e = Deque(d) - self.assertEqual(type(d), type(e)) - self.assertEqual(list(d), list(e)) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - s = pickle.dumps(d, proto) - e = pickle.loads(s) - self.assertNotEqual(id(d), id(e)) - self.assertEqual(type(d), type(e)) - self.assertEqual(list(d), list(e)) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + s = pickle.dumps(d, proto) + e = pickle.loads(s) + self.assertNotEqual(id(d), id(e)) + self.assertEqual(type(d), type(e)) + self.assertEqual(list(d), list(e)) + self.assertEqual(e.x, d.x) + self.assertEqual(e.z, d.z) + self.assertFalse(hasattr(e, 'y')) def test_pickle_recursive(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): diff --git a/Lib/test/test_descrtut.py b/Lib/test/test_descrtut.py index 2af683e707c..e01a31a7469 100644 --- a/Lib/test/test_descrtut.py +++ b/Lib/test/test_descrtut.py @@ -181,6 +181,7 @@ def merge(self, other): '__ge__', '__getattribute__', '__getitem__', + '__getstate__', '__gt__', '__hash__', '__iadd__', diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index d51296a2d12..37447fd249b 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -287,6 +287,8 @@ def test_copying(self): # and have a repr/eval round-trip pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] od = OrderedDict(pairs) + od.x = ['x'] + od.z = ['z'] def check(dup): msg = "\ncopy: %s\nod: %s" % (dup, od) self.assertIsNot(dup, od, msg) @@ -295,13 +297,27 @@ def check(dup): self.assertEqual(len(dup), len(od)) self.assertEqual(type(dup), type(od)) check(od.copy()) - check(copy.copy(od)) - check(copy.deepcopy(od)) + dup = copy.copy(od) + check(dup) + self.assertIs(dup.x, od.x) + self.assertIs(dup.z, od.z) + self.assertFalse(hasattr(dup, 'y')) + dup = copy.deepcopy(od) + check(dup) + self.assertEqual(dup.x, od.x) + self.assertIsNot(dup.x, od.x) + self.assertEqual(dup.z, od.z) + self.assertIsNot(dup.z, od.z) + self.assertFalse(hasattr(dup, 'y')) # pickle directly pulls the module, so we have to fake it with replaced_module('collections', self.module): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.subTest(proto=proto): - check(pickle.loads(pickle.dumps(od, proto))) + dup = pickle.loads(pickle.dumps(od, proto)) + check(dup) + self.assertEqual(dup.x, od.x) + self.assertEqual(dup.z, od.z) + self.assertFalse(hasattr(dup, 'y')) check(eval(repr(od))) update_test = OrderedDict() update_test.update(od) @@ -846,6 +862,23 @@ class OrderedDict(c_coll.OrderedDict): pass +class PurePythonOrderedDictWithSlotsCopyingTests(unittest.TestCase): + + module = py_coll + class OrderedDict(py_coll.OrderedDict): + __slots__ = ('x', 'y') + test_copying = OrderedDictTests.test_copying + + +@unittest.skipUnless(c_coll, 'requires the C version of the collections module') +class CPythonOrderedDictWithSlotsCopyingTests(unittest.TestCase): + + module = c_coll + class OrderedDict(c_coll.OrderedDict): + __slots__ = ('x', 'y') + test_copying = OrderedDictTests.test_copying + + class PurePythonGeneralMappingTests(mapping_tests.BasicTestMappingProtocol): @classmethod diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index 03b911920e1..3b57517a861 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -227,14 +227,17 @@ def test_sub_and_super(self): def test_pickling(self): for i in range(pickle.HIGHEST_PROTOCOL + 1): + if type(self.s) not in (set, frozenset): + self.s.x = ['x'] + self.s.z = ['z'] p = pickle.dumps(self.s, i) dup = pickle.loads(p) self.assertEqual(self.s, dup, "%s != %s" % (self.s, dup)) if type(self.s) not in (set, frozenset): - self.s.x = 10 - p = pickle.dumps(self.s, i) - dup = pickle.loads(p) self.assertEqual(self.s.x, dup.x) + self.assertEqual(self.s.z, dup.z) + self.assertFalse(hasattr(self.s, 'y')) + del self.s.x, self.s.z def test_iterator_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -808,6 +811,21 @@ def test_singleton_empty_frozenset(self): # All empty frozenset subclass instances should have different ids self.assertEqual(len(set(map(id, efs))), len(efs)) + +class SetSubclassWithSlots(set): + __slots__ = ('x', 'y', '__dict__') + +class TestSetSubclassWithSlots(unittest.TestCase): + thetype = SetSubclassWithSlots + setUp = TestJointOps.setUp + test_pickling = TestJointOps.test_pickling + +class FrozenSetSubclassWithSlots(frozenset): + __slots__ = ('x', 'y', '__dict__') + +class TestFrozenSetSubclassWithSlots(TestSetSubclassWithSlots): + thetype = FrozenSetSubclassWithSlots + # Tests taken from test_sets.py ============================================= empty_set = set() diff --git a/Lib/test/test_weakset.py b/Lib/test/test_weakset.py index 9b31d5fce34..d6c88596cff 100644 --- a/Lib/test/test_weakset.py +++ b/Lib/test/test_weakset.py @@ -1,5 +1,6 @@ import unittest from weakref import WeakSet +import copy import string from collections import UserString as ustr from collections.abc import Set, MutableSet @@ -15,6 +16,12 @@ class RefCycle: def __init__(self): self.cycle = self +class WeakSetSubclass(WeakSet): + pass + +class WeakSetWithSlots(WeakSet): + __slots__ = ('x', 'y') + class TestWeakSet(unittest.TestCase): @@ -447,6 +454,30 @@ def test_abc(self): self.assertIsInstance(self.s, Set) self.assertIsInstance(self.s, MutableSet) + def test_copying(self): + for cls in WeakSet, WeakSetWithSlots: + s = cls(self.items) + s.x = ['x'] + s.z = ['z'] + + dup = copy.copy(s) + self.assertIsInstance(dup, cls) + self.assertEqual(dup, s) + self.assertIsNot(dup, s) + self.assertIs(dup.x, s.x) + self.assertIs(dup.z, s.z) + self.assertFalse(hasattr(dup, 'y')) + + dup = copy.deepcopy(s) + self.assertIsInstance(dup, cls) + self.assertEqual(dup, s) + self.assertIsNot(dup, s) + self.assertEqual(dup.x, s.x) + self.assertIsNot(dup.x, s.x) + self.assertEqual(dup.z, s.z) + self.assertIsNot(dup.z, s.z) + self.assertFalse(hasattr(dup, 'y')) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index d2bdc4f7f04..60a41506d87 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -2524,8 +2524,7 @@ def test_pickle_issue18997(self): 4 """ e1 = dumper.fromstring(XMLTEXT) - if hasattr(e1, '__getstate__'): - self.assertEqual(e1.__getstate__()['tag'], 'group') + self.assertEqual(e1.__getstate__()['tag'], 'group') e2 = self.pickleRoundTrip(e1, 'xml.etree.ElementTree', dumper, loader, proto) self.assertEqual(e2.tag, 'group') diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-07-23-11-28-45.bpo-26579.lpCY8R.rst b/Misc/NEWS.d/next/Core and Builtins/2017-07-23-11-28-45.bpo-26579.lpCY8R.rst new file mode 100644 index 00000000000..9afd1bfa500 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-07-23-11-28-45.bpo-26579.lpCY8R.rst @@ -0,0 +1,7 @@ +Added ``object.__getstate__`` which provides the default implementation of +the ``__getstate__()`` method. + +Copying and pickling instances of subclasses of builtin types bytearray, +set, frozenset, collections.OrderedDict, collections.deque, weakref.WeakSet, +and datetime.tzinfo now copies and pickles instance attributes implemented as +slots. diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index b47f9777317..f78e2613fc2 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1347,27 +1347,24 @@ deque_traverse(dequeobject *deque, visitproc visit, void *arg) static PyObject * deque_reduce(dequeobject *deque, PyObject *Py_UNUSED(ignored)) { - PyObject *dict, *it; + PyObject *state, *it; - if (_PyObject_LookupAttr((PyObject *)deque, &_Py_ID(__dict__), &dict) < 0) { + state = _PyObject_GetState((PyObject *)deque); + if (state == NULL) { return NULL; } - if (dict == NULL) { - dict = Py_None; - Py_INCREF(dict); - } it = PyObject_GetIter((PyObject *)deque); if (it == NULL) { - Py_DECREF(dict); + Py_DECREF(state); return NULL; } if (deque->maxlen < 0) { - return Py_BuildValue("O()NN", Py_TYPE(deque), dict, it); + return Py_BuildValue("O()NN", Py_TYPE(deque), state, it); } else { - return Py_BuildValue("O(()n)NN", Py_TYPE(deque), deque->maxlen, dict, it); + return Py_BuildValue("O(()n)NN", Py_TYPE(deque), deque->maxlen, state, it); } } diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index ae97190bccb..fc766c3c943 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -3736,9 +3736,8 @@ static PyObject * tzinfo_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *args, *state; - PyObject *getinitargs, *getstate; + PyObject *getinitargs; _Py_IDENTIFIER(__getinitargs__); - _Py_IDENTIFIER(__getstate__); if (_PyObject_LookupAttrId(self, &PyId___getinitargs__, &getinitargs) < 0) { return NULL; @@ -3754,34 +3753,13 @@ tzinfo_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) return NULL; } - if (_PyObject_LookupAttrId(self, &PyId___getstate__, &getstate) < 0) { + state = _PyObject_GetState(self); + if (state == NULL) { Py_DECREF(args); return NULL; } - if (getstate != NULL) { - state = PyObject_CallNoArgs(getstate); - Py_DECREF(getstate); - if (state == NULL) { - Py_DECREF(args); - return NULL; - } - } - else { - PyObject **dictptr; - state = Py_None; - dictptr = _PyObject_GetDictPtr(self); - if (dictptr && *dictptr && PyDict_GET_SIZE(*dictptr)) { - state = *dictptr; - } - Py_INCREF(state); - } - if (state == Py_None) { - Py_DECREF(state); - return Py_BuildValue("(ON)", Py_TYPE(self), args); - } - else - return Py_BuildValue("(ONN)", Py_TYPE(self), args, state); + return Py_BuildValue("(ONN)", Py_TYPE(self), args, state); } static PyMethodDef tzinfo_methods[] = { diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 3849337fbf7..f784e044819 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2122,35 +2122,26 @@ bytearray_hex_impl(PyByteArrayObject *self, PyObject *sep, int bytes_per_sep) static PyObject * _common_reduce(PyByteArrayObject *self, int proto) { - PyObject *dict; - char *buf; + PyObject *state; + const char *buf; - if (_PyObject_LookupAttr((PyObject *)self, &_Py_ID(__dict__), &dict) < 0) { + state = _PyObject_GetState((PyObject *)self); + if (state == NULL) { return NULL; } - if (dict == NULL) { - dict = Py_None; - Py_INCREF(dict); - } + if (!Py_SIZE(self)) { + return Py_BuildValue("(O()N)", Py_TYPE(self), state); + } buf = PyByteArray_AS_STRING(self); if (proto < 3) { /* use str based reduction for backwards compatibility with Python 2.x */ - PyObject *latin1; - if (Py_SIZE(self)) - latin1 = PyUnicode_DecodeLatin1(buf, Py_SIZE(self), NULL); - else - latin1 = PyUnicode_FromString(""); - return Py_BuildValue("(O(Ns)N)", Py_TYPE(self), latin1, "latin-1", dict); + PyObject *latin1 = PyUnicode_DecodeLatin1(buf, Py_SIZE(self), NULL); + return Py_BuildValue("(O(Ns)N)", Py_TYPE(self), latin1, "latin-1", state); } else { /* use more efficient byte based reduction */ - if (Py_SIZE(self)) { - return Py_BuildValue("(O(y#)N)", Py_TYPE(self), buf, Py_SIZE(self), dict); - } - else { - return Py_BuildValue("(O()N)", Py_TYPE(self), dict); - } + return Py_BuildValue("(O(y#)N)", Py_TYPE(self), buf, Py_SIZE(self), state); } } diff --git a/Objects/clinic/typeobject.c.h b/Objects/clinic/typeobject.c.h index 8c70d76d916..dee3139bd3d 100644 --- a/Objects/clinic/typeobject.c.h +++ b/Objects/clinic/typeobject.c.h @@ -130,6 +130,24 @@ type___sizeof__(PyTypeObject *self, PyObject *Py_UNUSED(ignored)) return type___sizeof___impl(self); } +PyDoc_STRVAR(object___getstate____doc__, +"__getstate__($self, /)\n" +"--\n" +"\n" +"Helper for pickle."); + +#define OBJECT___GETSTATE___METHODDEF \ + {"__getstate__", (PyCFunction)object___getstate__, METH_NOARGS, object___getstate____doc__}, + +static PyObject * +object___getstate___impl(PyObject *self); + +static PyObject * +object___getstate__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return object___getstate___impl(self); +} + PyDoc_STRVAR(object___reduce____doc__, "__reduce__($self, /)\n" "--\n" @@ -243,4 +261,4 @@ object___dir__(PyObject *self, PyObject *Py_UNUSED(ignored)) { return object___dir___impl(self); } -/*[clinic end generated code: output=b4fb62939b08baf9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a30090032b8e6195 input=a9049054013a1b77]*/ diff --git a/Objects/odictobject.c b/Objects/odictobject.c index c207593ab79..f5b8b3e6cdd 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -947,23 +947,13 @@ PyDoc_STRVAR(odict_reduce__doc__, "Return state information for pickling"); static PyObject * odict_reduce(register PyODictObject *od, PyObject *Py_UNUSED(ignored)) { - PyObject *dict = NULL, *result = NULL; + PyObject *state, *result = NULL; PyObject *items_iter, *items, *args = NULL; /* capture any instance state */ - dict = PyObject_GetAttr((PyObject *)od, &_Py_ID(__dict__)); - if (dict == NULL) + state = _PyObject_GetState((PyObject *)od); + if (state == NULL) goto Done; - else { - /* od.__dict__ isn't necessarily a dict... */ - Py_ssize_t dict_len = PyObject_Length(dict); - if (dict_len == -1) - goto Done; - if (!dict_len) { - /* nothing to pickle in od.__dict__ */ - Py_CLEAR(dict); - } - } /* build the result */ args = PyTuple_New(0); @@ -979,11 +969,11 @@ odict_reduce(register PyODictObject *od, PyObject *Py_UNUSED(ignored)) if (items_iter == NULL) goto Done; - result = PyTuple_Pack(5, Py_TYPE(od), args, dict ? dict : Py_None, Py_None, items_iter); + result = PyTuple_Pack(5, Py_TYPE(od), args, state, Py_None, items_iter); Py_DECREF(items_iter); Done: - Py_XDECREF(dict); + Py_XDECREF(state); Py_XDECREF(args); return result; diff --git a/Objects/setobject.c b/Objects/setobject.c index ef2190de914..4b6a8b8dfb6 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1947,7 +1947,7 @@ an exception when an element is missing from the set."); static PyObject * set_reduce(PySetObject *so, PyObject *Py_UNUSED(ignored)) { - PyObject *keys=NULL, *args=NULL, *result=NULL, *dict=NULL; + PyObject *keys=NULL, *args=NULL, *result=NULL, *state=NULL; keys = PySequence_List((PyObject *)so); if (keys == NULL) @@ -1955,18 +1955,14 @@ set_reduce(PySetObject *so, PyObject *Py_UNUSED(ignored)) args = PyTuple_Pack(1, keys); if (args == NULL) goto done; - if (_PyObject_LookupAttr((PyObject *)so, &_Py_ID(__dict__), &dict) < 0) { + state = _PyObject_GetState((PyObject *)so); + if (state == NULL) goto done; - } - if (dict == NULL) { - dict = Py_None; - Py_INCREF(dict); - } - result = PyTuple_Pack(3, Py_TYPE(so), args, dict); + result = PyTuple_Pack(3, Py_TYPE(so), args, state); done: Py_XDECREF(args); Py_XDECREF(keys); - Py_XDECREF(dict); + Py_XDECREF(state); return result; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 5de8c3d2ece..53e4f0781d6 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4960,143 +4960,175 @@ _PyType_GetSlotNames(PyTypeObject *cls) } static PyObject * -_PyObject_GetState(PyObject *obj, int required) +object_getstate_default(PyObject *obj, int required) { PyObject *state; - PyObject *getstate; + PyObject *slotnames; - if (_PyObject_LookupAttr(obj, &_Py_ID(__getstate__), &getstate) < 0) { + if (required && Py_TYPE(obj)->tp_itemsize) { + PyErr_Format(PyExc_TypeError, + "cannot pickle %.200s objects", + Py_TYPE(obj)->tp_name); return NULL; } - if (getstate == NULL) { - PyObject *slotnames; - if (required && Py_TYPE(obj)->tp_itemsize) { + if (_PyObject_IsInstanceDictEmpty(obj)) { + state = Py_None; + Py_INCREF(state); + } + else { + state = PyObject_GenericGetDict(obj, NULL); + if (state == NULL) { + return NULL; + } + } + + slotnames = _PyType_GetSlotNames(Py_TYPE(obj)); + if (slotnames == NULL) { + Py_DECREF(state); + return NULL; + } + + assert(slotnames == Py_None || PyList_Check(slotnames)); + if (required) { + Py_ssize_t basicsize = PyBaseObject_Type.tp_basicsize; + if (Py_TYPE(obj)->tp_dictoffset && + (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) + { + basicsize += sizeof(PyObject *); + } + if (Py_TYPE(obj)->tp_weaklistoffset) { + basicsize += sizeof(PyObject *); + } + if (slotnames != Py_None) { + basicsize += sizeof(PyObject *) * PyList_GET_SIZE(slotnames); + } + if (Py_TYPE(obj)->tp_basicsize > basicsize) { + Py_DECREF(slotnames); + Py_DECREF(state); PyErr_Format(PyExc_TypeError, "cannot pickle '%.200s' object", Py_TYPE(obj)->tp_name); return NULL; } - if (_PyObject_IsInstanceDictEmpty(obj)) { - state = Py_None; - Py_INCREF(state); - } - else { - state = PyObject_GenericGetDict(obj, NULL); - if (state == NULL) { - return NULL; - } - } + } - slotnames = _PyType_GetSlotNames(Py_TYPE(obj)); - if (slotnames == NULL) { + if (slotnames != Py_None && PyList_GET_SIZE(slotnames) > 0) { + PyObject *slots; + Py_ssize_t slotnames_size, i; + + slots = PyDict_New(); + if (slots == NULL) { + Py_DECREF(slotnames); Py_DECREF(state); return NULL; } - assert(slotnames == Py_None || PyList_Check(slotnames)); - if (required) { - Py_ssize_t basicsize = PyBaseObject_Type.tp_basicsize; - if (Py_TYPE(obj)->tp_dictoffset && - (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) - { - basicsize += sizeof(PyObject *); + slotnames_size = PyList_GET_SIZE(slotnames); + for (i = 0; i < slotnames_size; i++) { + PyObject *name, *value; + + name = PyList_GET_ITEM(slotnames, i); + Py_INCREF(name); + value = PyObject_GetAttr(obj, name); + if (_PyObject_LookupAttr(obj, name, &value) < 0) { + Py_DECREF(name); + goto error; } - if (Py_TYPE(obj)->tp_weaklistoffset) { - basicsize += sizeof(PyObject *); + if (value == NULL) { + Py_DECREF(name); + /* It is not an error if the attribute is not present. */ } - if (slotnames != Py_None) { - basicsize += sizeof(PyObject *) * PyList_GET_SIZE(slotnames); + else { + int err = PyDict_SetItem(slots, name, value); + Py_DECREF(name); + Py_DECREF(value); + if (err) { + goto error; + } } - if (Py_TYPE(obj)->tp_basicsize > basicsize) { + + /* The list is stored on the class so it may mutate while we + iterate over it */ + if (slotnames_size != PyList_GET_SIZE(slotnames)) { + PyErr_Format(PyExc_RuntimeError, + "__slotsname__ changed size during iteration"); + goto error; + } + + /* We handle errors within the loop here. */ + if (0) { + error: Py_DECREF(slotnames); + Py_DECREF(slots); Py_DECREF(state); - PyErr_Format(PyExc_TypeError, - "cannot pickle '%.200s' object", - Py_TYPE(obj)->tp_name); return NULL; } } - if (slotnames != Py_None && PyList_GET_SIZE(slotnames) > 0) { - PyObject *slots; - Py_ssize_t slotnames_size, i; + /* If we found some slot attributes, pack them in a tuple along + the original attribute dictionary. */ + if (PyDict_GET_SIZE(slots) > 0) { + PyObject *state2; - slots = PyDict_New(); - if (slots == NULL) { + state2 = PyTuple_Pack(2, state, slots); + Py_DECREF(state); + if (state2 == NULL) { Py_DECREF(slotnames); - Py_DECREF(state); + Py_DECREF(slots); return NULL; } - - slotnames_size = PyList_GET_SIZE(slotnames); - for (i = 0; i < slotnames_size; i++) { - PyObject *name, *value; - - name = PyList_GET_ITEM(slotnames, i); - Py_INCREF(name); - if (_PyObject_LookupAttr(obj, name, &value) < 0) { - goto error; - } - if (value == NULL) { - Py_DECREF(name); - /* It is not an error if the attribute is not present. */ - } - else { - int err = PyDict_SetItem(slots, name, value); - Py_DECREF(name); - Py_DECREF(value); - if (err) { - goto error; - } - } - - /* The list is stored on the class so it may mutate while we - iterate over it */ - if (slotnames_size != PyList_GET_SIZE(slotnames)) { - PyErr_Format(PyExc_RuntimeError, - "__slotsname__ changed size during iteration"); - goto error; - } - - /* We handle errors within the loop here. */ - if (0) { - error: - Py_DECREF(slotnames); - Py_DECREF(slots); - Py_DECREF(state); - return NULL; - } - } - - /* If we found some slot attributes, pack them in a tuple along - the original attribute dictionary. */ - if (PyDict_GET_SIZE(slots) > 0) { - PyObject *state2; - - state2 = PyTuple_Pack(2, state, slots); - Py_DECREF(state); - if (state2 == NULL) { - Py_DECREF(slotnames); - Py_DECREF(slots); - return NULL; - } - state = state2; - } - Py_DECREF(slots); + state = state2; } - Py_DECREF(slotnames); - } - else { /* getstate != NULL */ - state = _PyObject_CallNoArgs(getstate); - Py_DECREF(getstate); - if (state == NULL) - return NULL; + Py_DECREF(slots); } + Py_DECREF(slotnames); return state; } +static PyObject * +object_getstate(PyObject *obj, int required) +{ + PyObject *getstate, *state; + + getstate = PyObject_GetAttr(obj, &_Py_ID(__getstate__)); + if (getstate == NULL) { + return NULL; + } + if (PyCFunction_Check(getstate) && + PyCFunction_GET_SELF(getstate) == obj && + PyCFunction_GET_FUNCTION(getstate) == object___getstate__) + { + /* If __getstate__ is not overriden pass the required argument. */ + state = object_getstate_default(obj, required); + } + else { + state = _PyObject_CallNoArgs(getstate); + } + Py_DECREF(getstate); + return state; +} + +PyObject * +_PyObject_GetState(PyObject *obj) +{ + return object_getstate(obj, 0); +} + +/*[clinic input] +object.__getstate__ + +Helper for pickle. +[clinic start generated code]*/ + +static PyObject * +object___getstate___impl(PyObject *self) +/*[clinic end generated code: output=5a2500dcb6217e9e input=692314d8fbe194ee]*/ +{ + return object_getstate_default(self, 0); +} + static int _PyObject_GetNewArguments(PyObject *obj, PyObject **args, PyObject **kwargs) { @@ -5309,8 +5341,7 @@ reduce_newobj(PyObject *obj) return NULL; } - state = _PyObject_GetState(obj, - !hasargs && !PyList_Check(obj) && !PyDict_Check(obj)); + state = object_getstate(obj, !(hasargs || PyList_Check(obj) || PyDict_Check(obj))); if (state == NULL) { Py_DECREF(newobj); Py_DECREF(newargs); @@ -5558,6 +5589,7 @@ object___dir___impl(PyObject *self) static PyMethodDef object_methods[] = { OBJECT___REDUCE_EX___METHODDEF OBJECT___REDUCE___METHODDEF + OBJECT___GETSTATE___METHODDEF {"__subclasshook__", object_subclasshook, METH_CLASS | METH_VARARGS, object_subclasshook_doc}, {"__init_subclass__", object_init_subclass, METH_CLASS | METH_NOARGS,