gh-119127: functools.partial placeholders (gh-119827)

Doc/library/functools.rst

@@ -328,6 +328,14 @@ The :mod:`functools` module defines the following functions:
       Returning ``NotImplemented`` from the underlying comparison function for
       unrecognised types is now supported.
 
+.. data:: Placeholder
+
+   A singleton object used as a sentinel to reserve a place
+   for positional arguments when calling :func:`partial`
+   and :func:`partialmethod`.
+
+   .. versionadded:: 3.14
+
 .. function:: partial(func, /, *args, **keywords)
 
    Return a new :ref:`partial object<partial-objects>` which when called
@@ -338,26 +346,67 @@ The :mod:`functools` module defines the following functions:
    Roughly equivalent to::
 
       def partial(func, /, *args, **keywords):
-          def newfunc(*fargs, **fkeywords):
-              newkeywords = {**keywords, **fkeywords}
-              return func(*args, *fargs, **newkeywords)
+          def newfunc(*more_args, **more_keywords):
+              keywords_union = {**keywords, **more_keywords}
+              return func(*args, *more_args, **keywords_union)
           newfunc.func = func
           newfunc.args = args
           newfunc.keywords = keywords
           return newfunc
 
-   The :func:`partial` is used for partial function application which "freezes"
+   The :func:`partial` function is used for partial function application which "freezes"
    some portion of a function's arguments and/or keywords resulting in a new object
    with a simplified signature.  For example, :func:`partial` can be used to create
    a callable that behaves like the :func:`int` function where the *base* argument
-   defaults to two:
+   defaults to ``2``:
+
+   .. doctest::
 
-      >>> from functools import partial
       >>> basetwo = partial(int, base=2)
       >>> basetwo.__doc__ = 'Convert base 2 string to an int.'
       >>> basetwo('10010')
       18
 
+   If :data:`Placeholder` sentinels are present in *args*, they will be filled first
+   when :func:`partial` is called. This allows custom selection of positional arguments
+   to be pre-filled when constructing a :ref:`partial object <partial-objects>`.
+
+   If :data:`!Placeholder` sentinels are present, all of them must be filled at call time:
+
+   .. doctest::
+
+      >>> say_to_world = partial(print, Placeholder, Placeholder, "world!")
+      >>> say_to_world('Hello', 'dear')
+      Hello dear world!
+
+   Calling ``say_to_world('Hello')`` would raise a :exc:`TypeError`, because
+   only one positional argument is provided, while there are two placeholders
+   in :ref:`partial object <partial-objects>`.
+
+   Successive :func:`partial` applications fill :data:`!Placeholder` sentinels
+   of the input :func:`partial` objects with new positional arguments.
+   A place for positional argument can be retained by inserting new
+   :data:`!Placeholder` sentinel to the place held by previous :data:`!Placeholder`:
+
+   .. doctest::
+
+      >>> from functools import partial, Placeholder as _
+      >>> remove = partial(str.replace, _, _, '')
+      >>> message = 'Hello, dear dear world!'
+      >>> remove(message, ' dear')
+      'Hello, world!'
+      >>> remove_dear = partial(remove, _, ' dear')
+      >>> remove_dear(message)
+      'Hello, world!'
+      >>> remove_first_dear = partial(remove_dear, _, 1)
+      >>> remove_first_dear(message)
+      'Hello, dear world!'
+
+   Note, :data:`!Placeholder` has no special treatment when used for keyword
+   argument of :data:`!Placeholder`.
+
+   .. versionchanged:: 3.14
+      Added support for :data:`Placeholder` in positional arguments.
 
 .. class:: partialmethod(func, /, *args, **keywords)
 
@@ -742,10 +791,7 @@ have three read-only attributes:
    The keyword arguments that will be supplied when the :class:`partial` object is
    called.
 
-:class:`partial` objects are like :ref:`function objects <user-defined-funcs>`
-in that they are callable, weak referenceable, and can have attributes.
-There are some important differences.  For instance, the
-:attr:`~function.__name__` and :attr:`function.__doc__` attributes
-are not created automatically.  Also, :class:`partial` objects defined in
-classes behave like static methods and do not transform into bound methods
-during instance attribute look-up.
+:class:`partial` objects are like :class:`function` objects in that they are
+callable, weak referenceable, and can have attributes.  There are some important
+differences.  For instance, the :attr:`~definition.__name__` and :attr:`__doc__` attributes
+are not created automatically.

Doc/whatsnew/3.14.rst

@@ -255,6 +255,15 @@ Added support for converting any objects that have the
 (Contributed by Serhiy Storchaka in :gh:`82017`.)
 
 
+functools
+---------
+
+* Added support to :func:`functools.partial` and
+  :func:`functools.partialmethod` for :data:`functools.Placeholder` sentinels
+  to reserve a place for positional arguments.
+  (Contributed by Dominykas Grigonis in :gh:`119127`.)
+
+
 http
 ----
 

Lib/functools.py

@@ -6,17 +6,18 @@
 # Written by Nick Coghlan <ncoghlan at gmail.com>,
 # Raymond Hettinger <python at rcn.com>,
 # and Łukasz Langa <lukasz at langa.pl>.
-#   Copyright (C) 2006-2013 Python Software Foundation.
+#   Copyright (C) 2006-2024 Python Software Foundation.
 # See C source code for _functools credits/copyright
 
 __all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES',
            'total_ordering', 'cache', 'cmp_to_key', 'lru_cache', 'reduce',
            'partial', 'partialmethod', 'singledispatch', 'singledispatchmethod',
-           'cached_property']
+           'cached_property', 'Placeholder']
 
 from abc import get_cache_token
 from collections import namedtuple
 # import types, weakref  # Deferred to single_dispatch()
+from operator import itemgetter
 from reprlib import recursive_repr
 from types import MethodType
 from _thread import RLock
@@ -274,43 +275,125 @@ def reduce(function, sequence, initial=_initial_missing):
 ### partial() argument application
 ################################################################################
 
+
+class _PlaceholderType:
+    """The type of the Placeholder singleton.
+
+    Used as a placeholder for partial arguments.
+    """
+    __instance = None
+    __slots__ = ()
+
+    def __init_subclass__(cls, *args, **kwargs):
+        raise TypeError(f"type '{cls.__name__}' is not an acceptable base type")
+
+    def __new__(cls):
+        if cls.__instance is None:
+            cls.__instance = object.__new__(cls)
+        return cls.__instance
+
+    def __repr__(self):
+        return 'Placeholder'
+
+    def __reduce__(self):
+        return 'Placeholder'
+
+Placeholder = _PlaceholderType()
+
+def _partial_prepare_merger(args):
+    if not args:
+        return 0, None
+    nargs = len(args)
+    order = []
+    j = nargs
+    for i, a in enumerate(args):
+        if a is Placeholder:
+            order.append(j)
+            j += 1
+        else:
+            order.append(i)
+    phcount = j - nargs
+    merger = itemgetter(*order) if phcount else None
+    return phcount, merger
+
+def _partial_new(cls, func, /, *args, **keywords):
+    if issubclass(cls, partial):
+        base_cls = partial
+        if not callable(func):
+            raise TypeError("the first argument must be callable")
+    else:
+        base_cls = partialmethod
+        # func could be a descriptor like classmethod which isn't callable
+        if not callable(func) and not hasattr(func, "__get__"):
+            raise TypeError(f"the first argument {func!r} must be a callable "
+                            "or a descriptor")
+    if args and args[-1] is Placeholder:
+        raise TypeError("trailing Placeholders are not allowed")
+    if isinstance(func, base_cls):
+        pto_phcount = func._phcount
+        tot_args = func.args
+        if args:
+            tot_args += args
+            if pto_phcount:
+                # merge args with args of `func` which is `partial`
+                nargs = len(args)
+                if nargs < pto_phcount:
+                    tot_args += (Placeholder,) * (pto_phcount - nargs)
+                tot_args = func._merger(tot_args)
+                if nargs > pto_phcount:
+                    tot_args += args[pto_phcount:]
+            phcount, merger = _partial_prepare_merger(tot_args)
+        else:   # works for both pto_phcount == 0 and != 0
+            phcount, merger = pto_phcount, func._merger
+        keywords = {**func.keywords, **keywords}
+        func = func.func
+    else:
+        tot_args = args
+        phcount, merger = _partial_prepare_merger(tot_args)
+
+    self = object.__new__(cls)
+    self.func = func
+    self.args = tot_args
+    self.keywords = keywords
+    self._phcount = phcount
+    self._merger = merger
+    return self
+
+def _partial_repr(self):
+    cls = type(self)
+    module = cls.__module__
+    qualname = cls.__qualname__
+    args = [repr(self.func)]
+    args.extend(map(repr, self.args))
+    args.extend(f"{k}={v!r}" for k, v in self.keywords.items())
+    return f"{module}.{qualname}({', '.join(args)})"
+
 # Purely functional, no descriptor behaviour
 class partial:
     """New function with partial application of the given arguments
     and keywords.
     """
 
-    __slots__ = "func", "args", "keywords", "__dict__", "__weakref__"
+    __slots__ = ("func", "args", "keywords", "_phcount", "_merger",
+                 "__dict__", "__weakref__")
 
-    def __new__(cls, func, /, *args, **keywords):
-        if not callable(func):
-            raise TypeError("the first argument must be callable")
-
-        if isinstance(func, partial):
-            args = func.args + args
-            keywords = {**func.keywords, **keywords}
-            func = func.func
-
-        self = super(partial, cls).__new__(cls)
-
-        self.func = func
-        self.args = args
-        self.keywords = keywords
-        return self
+    __new__ = _partial_new
+    __repr__ = recursive_repr()(_partial_repr)
 
     def __call__(self, /, *args, **keywords):
+        phcount = self._phcount
+        if phcount:
+            try:
+                pto_args = self._merger(self.args + args)
+                args = args[phcount:]
+            except IndexError:
+                raise TypeError("missing positional arguments "
+                                "in 'partial' call; expected "
+                                f"at least {phcount}, got {len(args)}")
+        else:
+            pto_args = self.args
         keywords = {**self.keywords, **keywords}
-        return self.func(*self.args, *args, **keywords)
-
-    @recursive_repr()
-    def __repr__(self):
-        cls = type(self)
-        qualname = cls.__qualname__
-        module = cls.__module__
-        args = [repr(self.func)]
-        args.extend(repr(x) for x in self.args)
-        args.extend(f"{k}={v!r}" for (k, v) in self.keywords.items())
-        return f"{module}.{qualname}({', '.join(args)})"
+        return self.func(*pto_args, *args, **keywords)
 
     def __get__(self, obj, objtype=None):
         if obj is None:
@@ -332,6 +415,10 @@ def __setstate__(self, state):
            (namespace is not None and not isinstance(namespace, dict))):
             raise TypeError("invalid partial state")
 
+        if args and args[-1] is Placeholder:
+            raise TypeError("trailing Placeholders are not allowed")
+        phcount, merger = _partial_prepare_merger(args)
+
         args = tuple(args) # just in case it's a subclass
         if kwds is None:
             kwds = {}
@@ -344,53 +431,40 @@ def __setstate__(self, state):
         self.func = func
         self.args = args
         self.keywords = kwds
+        self._phcount = phcount
+        self._merger = merger
 
 try:
-    from _functools import partial
+    from _functools import partial, Placeholder, _PlaceholderType
 except ImportError:
     pass
 
 # Descriptor version
-class partialmethod(object):
+class partialmethod:
     """Method descriptor with partial application of the given arguments
     and keywords.
 
     Supports wrapping existing descriptors and handles non-descriptor
     callables as instance methods.
     """
-
-    def __init__(self, func, /, *args, **keywords):
-        if not callable(func) and not hasattr(func, "__get__"):
-            raise TypeError("{!r} is not callable or a descriptor"
-                                 .format(func))
-
-        # func could be a descriptor like classmethod which isn't callable,
-        # so we can't inherit from partial (it verifies func is callable)
-        if isinstance(func, partialmethod):
-            # flattening is mandatory in order to place cls/self before all
-            # other arguments
-            # it's also more efficient since only one function will be called
-            self.func = func.func
-            self.args = func.args + args
-            self.keywords = {**func.keywords, **keywords}
-        else:
-            self.func = func
-            self.args = args
-            self.keywords = keywords
-
-    def __repr__(self):
-        cls = type(self)
-        module = cls.__module__
-        qualname = cls.__qualname__
-        args = [repr(self.func)]
-        args.extend(map(repr, self.args))
-        args.extend(f"{k}={v!r}" for k, v in self.keywords.items())
-        return f"{module}.{qualname}({', '.join(args)})"
+    __new__ = _partial_new
+    __repr__ = _partial_repr
 
     def _make_unbound_method(self):
         def _method(cls_or_self, /, *args, **keywords):
+            phcount = self._phcount
+            if phcount:
+                try:
+                    pto_args = self._merger(self.args + args)
+                    args = args[phcount:]
+                except IndexError:
+                    raise TypeError("missing positional arguments "
+                                    "in 'partialmethod' call; expected "
+                                    f"at least {phcount}, got {len(args)}")
+            else:
+                pto_args = self.args
             keywords = {**self.keywords, **keywords}
-            return self.func(cls_or_self, *self.args, *args, **keywords)
+            return self.func(cls_or_self, *pto_args, *args, **keywords)
         _method.__isabstractmethod__ = self.__isabstractmethod__
         _method.__partialmethod__ = self
         return _method

Lib/inspect.py

@@ -1930,7 +1930,12 @@ def _signature_get_partial(wrapped_sig, partial, extra_args=()):
             if param.kind is _POSITIONAL_ONLY:
                 # If positional-only parameter is bound by partial,
                 # it effectively disappears from the signature
-                new_params.pop(param_name)
+                # However, if it is a Placeholder it is not removed
+                # And also looses default value
+                if arg_value is functools.Placeholder:
+                    new_params[param_name] = param.replace(default=_empty)
+                else:
+                    new_params.pop(param_name)
                 continue
 
             if param.kind is _POSITIONAL_OR_KEYWORD:
@@ -1952,7 +1957,17 @@ def _signature_get_partial(wrapped_sig, partial, extra_args=()):
                     new_params[param_name] = param.replace(default=arg_value)
                 else:
                     # was passed as a positional argument
-                    new_params.pop(param.name)
+                    # Do not pop if it is a Placeholder
+                    #   also change kind to positional only
+                    #   and remove default
+                    if arg_value is functools.Placeholder:
+                        new_param = param.replace(
+                            kind=_POSITIONAL_ONLY,
+                            default=_empty
+                        )
+                        new_params[param_name] = new_param
+                    else:
+                        new_params.pop(param_name)
                     continue
 
             if param.kind is _KEYWORD_ONLY:
@@ -2446,6 +2461,11 @@ def _signature_from_callable(obj, *,
                 sig_params = tuple(sig.parameters.values())
                 assert (not sig_params or
                         first_wrapped_param is not sig_params[0])
+                # If there were placeholders set,
+                #   first param is transformed to positional only
+                if partialmethod.args.count(functools.Placeholder):
+                    first_wrapped_param = first_wrapped_param.replace(
+                        kind=Parameter.POSITIONAL_ONLY)
                 new_params = (first_wrapped_param,) + sig_params
                 return sig.replace(parameters=new_params)
 

Lib/test/test_functools.py

@@ -6,6 +6,7 @@
 from itertools import permutations
 import pickle
 from random import choice
+import re
 import sys
 from test import support
 import threading
@@ -210,6 +211,51 @@ def foo(bar):
         p2.new_attr = 'spam'
         self.assertEqual(p2.new_attr, 'spam')
 
+    def test_placeholders_trailing_raise(self):
+        PH = self.module.Placeholder
+        for args in [(PH,), (0, PH), (0, PH, 1, PH, PH, PH)]:
+            with self.assertRaises(TypeError):
+                self.partial(capture, *args)
+
+    def test_placeholders(self):
+        PH = self.module.Placeholder
+        # 1 Placeholder
+        args = (PH, 0)
+        p = self.partial(capture, *args)
+        actual_args, actual_kwds = p('x')
+        self.assertEqual(actual_args, ('x', 0))
+        self.assertEqual(actual_kwds, {})
+        # 2 Placeholders
+        args = (PH, 0, PH, 1)
+        p = self.partial(capture, *args)
+        with self.assertRaises(TypeError):
+            p('x')
+        actual_args, actual_kwds = p('x', 'y')
+        self.assertEqual(actual_args, ('x', 0, 'y', 1))
+        self.assertEqual(actual_kwds, {})
+
+    def test_placeholders_optimization(self):
+        PH = self.module.Placeholder
+        p = self.partial(capture, PH, 0)
+        p2 = self.partial(p, PH, 1, 2, 3)
+        self.assertEqual(p2.args, (PH, 0, 1, 2, 3))
+        p3 = self.partial(p2, -1, 4)
+        actual_args, actual_kwds = p3(5)
+        self.assertEqual(actual_args, (-1, 0, 1, 2, 3, 4, 5))
+        self.assertEqual(actual_kwds, {})
+        # inner partial has placeholders and outer partial has no args case
+        p = self.partial(capture, PH, 0)
+        p2 = self.partial(p)
+        self.assertEqual(p2.args, (PH, 0))
+        self.assertEqual(p2(1), ((1, 0), {}))
+
+    def test_construct_placeholder_singleton(self):
+        PH = self.module.Placeholder
+        tp = type(PH)
+        self.assertIs(tp(), PH)
+        self.assertRaises(TypeError, tp, 1, 2)
+        self.assertRaises(TypeError, tp, a=1, b=2)
+
     def test_repr(self):
         args = (object(), object())
         args_repr = ', '.join(repr(a) for a in args)
@@ -311,6 +357,23 @@ def test_setstate(self):
         self.assertEqual(f(2), ((2,), {}))
         self.assertEqual(f(), ((), {}))
 
+        # Set State with placeholders
+        PH = self.module.Placeholder
+        f = self.partial(signature)
+        f.__setstate__((capture, (PH, 1), dict(a=10), dict(attr=[])))
+        self.assertEqual(signature(f), (capture, (PH, 1), dict(a=10), dict(attr=[])))
+        msg_regex = re.escape("missing positional arguments in 'partial' call; "
+                              "expected at least 1, got 0")
+        with self.assertRaisesRegex(TypeError, f'^{msg_regex}$') as cm:
+            f()
+        self.assertEqual(f(2), ((2, 1), dict(a=10)))
+
+        # Trailing Placeholder error
+        f = self.partial(signature)
+        msg_regex = re.escape("trailing Placeholders are not allowed")
+        with self.assertRaisesRegex(TypeError, f'^{msg_regex}$') as cm:
+            f.__setstate__((capture, (1, PH), dict(a=10), dict(attr=[])))
+
     def test_setstate_errors(self):
         f = self.partial(signature)
         self.assertRaises(TypeError, f.__setstate__, (capture, (), {}))
@@ -456,6 +519,19 @@ def __str__(self):
         self.assertIn('astr', r)
         self.assertIn("['sth']", r)
 
+    def test_placeholders_refcount_smoke(self):
+        PH = self.module.Placeholder
+        # sum supports vector call
+        lst1, start = [], []
+        sum_lists = self.partial(sum, PH, start)
+        for i in range(10):
+            sum_lists([lst1, lst1])
+        # collections.ChainMap initializer does not support vectorcall
+        map1, map2 = {}, {}
+        partial_cm = self.partial(collections.ChainMap, PH, map1)
+        for i in range(10):
+            partial_cm(map2, map2)
+
 
 class TestPartialPy(TestPartial, unittest.TestCase):
     module = py_functools
@@ -480,6 +556,19 @@ class TestPartialCSubclass(TestPartialC):
 class TestPartialPySubclass(TestPartialPy):
     partial = PyPartialSubclass
 
+    def test_subclass_optimization(self):
+        # `partial` input to `partial` subclass
+        p = py_functools.partial(min, 2)
+        p2 = self.partial(p, 1)
+        self.assertIs(p2.func, min)
+        self.assertEqual(p2(0), 0)
+        # `partial` subclass input to `partial` subclass
+        p = self.partial(min, 2)
+        p2 = self.partial(p, 1)
+        self.assertIs(p2.func, min)
+        self.assertEqual(p2(0), 0)
+
+
 class TestPartialMethod(unittest.TestCase):
 
     class A(object):
@@ -617,6 +706,20 @@ def f(a, b, /):
         p = functools.partial(f, 1)
         self.assertEqual(p(2), f(1, 2))
 
+    def test_subclass_optimization(self):
+        class PartialMethodSubclass(functools.partialmethod):
+            pass
+        # `partialmethod` input to `partialmethod` subclass
+        p = functools.partialmethod(min, 2)
+        p2 = PartialMethodSubclass(p, 1)
+        self.assertIs(p2.func, min)
+        self.assertEqual(p2.__get__(0)(), 0)
+        # `partialmethod` subclass input to `partialmethod` subclass
+        p = PartialMethodSubclass(min, 2)
+        p2 = PartialMethodSubclass(p, 1)
+        self.assertIs(p2.func, min)
+        self.assertEqual(p2.__get__(0)(), 0)
+
 
 class TestUpdateWrapper(unittest.TestCase):
 

Lib/test/test_inspect/test_inspect.py

@@ -3341,7 +3341,7 @@ def foo(cls, *, arg):
                           ...))
 
     def test_signature_on_partial(self):
-        from functools import partial
+        from functools import partial, Placeholder
 
         def test():
             pass
@@ -3396,6 +3396,25 @@ def test(a, b, *, c, d):
                            ('d', ..., ..., "keyword_only")),
                           ...))
 
+        # With Placeholder
+        self.assertEqual(self.signature(partial(test, Placeholder, 1)),
+                         ((('a', ..., ..., "positional_only"),
+                           ('c', ..., ..., "keyword_only"),
+                           ('d', ..., ..., "keyword_only")),
+                          ...))
+
+        self.assertEqual(self.signature(partial(test, Placeholder, 1, c=2)),
+                         ((('a', ..., ..., "positional_only"),
+                           ('c', 2, ..., "keyword_only"),
+                           ('d', ..., ..., "keyword_only")),
+                          ...))
+
+        # Ensure unittest.mock.ANY & similar do not get picked up as a Placeholder
+        self.assertEqual(self.signature(partial(test, unittest.mock.ANY, 1, c=2)),
+                         ((('c', 2, ..., "keyword_only"),
+                           ('d', ..., ..., "keyword_only")),
+                          ...))
+
         def test(a, *args, b, **kwargs):
             pass
 
@@ -3443,6 +3462,15 @@ def test(a, *args, b, **kwargs):
                            ('kwargs', ..., ..., "var_keyword")),
                           ...))
 
+        # With Placeholder
+        p = partial(test, Placeholder, Placeholder, 1, b=0, test=1)
+        self.assertEqual(self.signature(p),
+                         ((('a', ..., ..., "positional_only"),
+                           ('args', ..., ..., "var_positional"),
+                           ('b', 0, ..., "keyword_only"),
+                           ('kwargs', ..., ..., "var_keyword")),
+                          ...))
+
         def test(a, b, c:int) -> 42:
             pass
 
@@ -3547,6 +3575,34 @@ def foo(a, b, /, c, d, **kwargs):
                            ('kwargs', ..., ..., 'var_keyword')),
                          ...))
 
+        # Positional only With Placeholder
+        p = partial(foo, Placeholder, 1, c=0, d=1)
+        self.assertEqual(self.signature(p),
+                         ((('a', ..., ..., "positional_only"),
+                           ('c', 0, ..., "keyword_only"),
+                           ('d', 1, ..., "keyword_only"),
+                           ('kwargs', ..., ..., "var_keyword")),
+                          ...))
+
+        # Optionals Positional With Placeholder
+        def foo(a=0, b=1, /, c=2, d=3):
+            pass
+
+        # Positional
+        p = partial(foo, Placeholder, 1, c=0, d=1)
+        self.assertEqual(self.signature(p),
+                         ((('a', ..., ..., "positional_only"),
+                           ('c', 0, ..., "keyword_only"),
+                           ('d', 1, ..., "keyword_only")),
+                          ...))
+
+        # Positional or Keyword - transformed to positional
+        p = partial(foo, Placeholder, 1, Placeholder, 1)
+        self.assertEqual(self.signature(p),
+                         ((('a', ..., ..., "positional_only"),
+                           ('c', ..., ..., "positional_only")),
+                          ...))
+
     def test_signature_on_partialmethod(self):
         from functools import partialmethod
 
@@ -3559,18 +3615,32 @@ def test():
             inspect.signature(Spam.ham)
 
         class Spam:
-            def test(it, a, *, c) -> 'spam':
+            def test(it, a, b, *, c) -> 'spam':
                 pass
             ham = partialmethod(test, c=1)
+            bar = partialmethod(test, functools.Placeholder, 1, c=1)
 
         self.assertEqual(self.signature(Spam.ham, eval_str=False),
                          ((('it', ..., ..., 'positional_or_keyword'),
                            ('a', ..., ..., 'positional_or_keyword'),
+                           ('b', ..., ..., 'positional_or_keyword'),
                            ('c', 1, ..., 'keyword_only')),
                           'spam'))
 
         self.assertEqual(self.signature(Spam().ham, eval_str=False),
                          ((('a', ..., ..., 'positional_or_keyword'),
+                           ('b', ..., ..., 'positional_or_keyword'),
+                           ('c', 1, ..., 'keyword_only')),
+                          'spam'))
+
+        # With Placeholder
+        self.assertEqual(self.signature(Spam.bar, eval_str=False),
+                         ((('it', ..., ..., 'positional_only'),
+                           ('a', ..., ..., 'positional_only'),
+                           ('c', 1, ..., 'keyword_only')),
+                          'spam'))
+        self.assertEqual(self.signature(Spam().bar, eval_str=False),
+                         ((('a', ..., ..., 'positional_only'),
                            ('c', 1, ..., 'keyword_only')),
                           'spam'))
 

Misc/NEWS.d/next/Library/2024-05-25-00-54-26.gh-issue-119127.LpPvag.rst

@@ -0,0 +1,2 @@
+Positional arguments of :func:`functools.partial` objects
+now support placeholders via :data:`functools.Placeholder`.

Modules/_functoolsmodule.c

@@ -25,6 +25,8 @@ class _functools._lru_cache_wrapper "PyObject *" "&lru_cache_type_spec"
 typedef struct _functools_state {
     /* this object is used delimit args and keywords in the cache keys */
     PyObject *kwd_mark;
+    PyTypeObject *placeholder_type;
+    PyObject *placeholder;
     PyTypeObject *partial_type;
     PyTypeObject *keyobject_type;
     PyTypeObject *lru_list_elem_type;
@@ -41,6 +43,79 @@ get_functools_state(PyObject *module)
 
 /* partial object **********************************************************/
 
+
+// The 'Placeholder' singleton indicates which formal positional
+// parameters are to be bound first when using a 'partial' object.
+
+typedef struct {
+    PyObject_HEAD
+} placeholderobject;
+
+static inline _functools_state *
+get_functools_state_by_type(PyTypeObject *type);
+
+PyDoc_STRVAR(placeholder_doc,
+"The type of the Placeholder singleton.\n\n"
+"Used as a placeholder for partial arguments.");
+
+static PyObject *
+placeholder_repr(PyObject *op)
+{
+    return PyUnicode_FromString("Placeholder");
+}
+
+static PyObject *
+placeholder_reduce(PyObject *op, PyObject *Py_UNUSED(ignored))
+{
+    return PyUnicode_FromString("Placeholder");
+}
+
+static PyMethodDef placeholder_methods[] = {
+    {"__reduce__", placeholder_reduce, METH_NOARGS, NULL},
+    {NULL, NULL}
+};
+
+static void
+placeholder_dealloc(PyObject* placeholder)
+{
+    /* This should never get called, but we also don't want to SEGV if
+     * we accidentally decref Placeholder out of existence. Instead,
+     * since Placeholder is an immortal object, re-set the reference count.
+     */
+    _Py_SetImmortal(placeholder);
+}
+
+static PyObject *
+placeholder_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
+{
+    if (PyTuple_GET_SIZE(args) || (kwargs && PyDict_GET_SIZE(kwargs))) {
+        PyErr_SetString(PyExc_TypeError, "PlaceholderType takes no arguments");
+        return NULL;
+    }
+    _functools_state *state = get_functools_state_by_type(type);
+    if (state->placeholder == NULL) {
+        state->placeholder = PyType_GenericNew(type, NULL, NULL);
+    }
+    return state->placeholder;
+}
+
+static PyType_Slot placeholder_type_slots[] = {
+    {Py_tp_dealloc, placeholder_dealloc},
+    {Py_tp_repr, placeholder_repr},
+    {Py_tp_doc, (void *)placeholder_doc},
+    {Py_tp_methods, placeholder_methods},
+    {Py_tp_new, placeholder_new},
+    {0, 0}
+};
+
+static PyType_Spec placeholder_type_spec = {
+    .name = "functools._PlaceholderType",
+    .basicsize = sizeof(placeholderobject),
+    .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE,
+    .slots = placeholder_type_slots
+};
+
+
 typedef struct {
     PyObject_HEAD
     PyObject *fn;
@@ -48,6 +123,8 @@ typedef struct {
     PyObject *kw;
     PyObject *dict;        /* __dict__ */
     PyObject *weakreflist; /* List of weak references */
+    PyObject *placeholder; /* Placeholder for positional arguments */
+    Py_ssize_t phcount;    /* Number of placeholders */
     vectorcallfunc vectorcall;
 } partialobject;
 
@@ -70,23 +147,38 @@ get_functools_state_by_type(PyTypeObject *type)
 static PyObject *
 partial_new(PyTypeObject *type, PyObject *args, PyObject *kw)
 {
-    PyObject *func, *pargs, *nargs, *pkw;
+    PyObject *func, *pto_args, *new_args, *pto_kw, *phold;
     partialobject *pto;
+    Py_ssize_t pto_phcount = 0;
+    Py_ssize_t new_nargs = PyTuple_GET_SIZE(args) - 1;
 
-    if (PyTuple_GET_SIZE(args) < 1) {
+    if (new_nargs < 0) {
         PyErr_SetString(PyExc_TypeError,
                         "type 'partial' takes at least one argument");
         return NULL;
     }
+    func = PyTuple_GET_ITEM(args, 0);
+    if (!PyCallable_Check(func)) {
+        PyErr_SetString(PyExc_TypeError,
+                        "the first argument must be callable");
+        return NULL;
+    }
 
     _functools_state *state = get_functools_state_by_type(type);
     if (state == NULL) {
         return NULL;
     }
+    phold = state->placeholder;
 
-    pargs = pkw = NULL;
-    func = PyTuple_GET_ITEM(args, 0);
+    /* Placeholder restrictions */
+    if (new_nargs && PyTuple_GET_ITEM(args, new_nargs) == phold) {
+        PyErr_SetString(PyExc_TypeError,
+                        "trailing Placeholders are not allowed");
+        return NULL;
+    }
 
+    /* check wrapped function / object */
+    pto_args = pto_kw = NULL;
     int res = PyObject_TypeCheck(func, state->partial_type);
     if (res == -1) {
         return NULL;
@@ -95,18 +187,14 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw)
         // We can use its underlying function directly and merge the arguments.
         partialobject *part = (partialobject *)func;
         if (part->dict == NULL) {
-            pargs = part->args;
-            pkw = part->kw;
+            pto_args = part->args;
+            pto_kw = part->kw;
             func = part->fn;
-            assert(PyTuple_Check(pargs));
-            assert(PyDict_Check(pkw));
+            pto_phcount = part->phcount;
+            assert(PyTuple_Check(pto_args));
+            assert(PyDict_Check(pto_kw));
         }
     }
-    if (!PyCallable_Check(func)) {
-        PyErr_SetString(PyExc_TypeError,
-                        "the first argument must be callable");
-        return NULL;
-    }
 
     /* create partialobject structure */
     pto = (partialobject *)type->tp_alloc(type, 0);
@@ -114,18 +202,58 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw)
         return NULL;
 
     pto->fn = Py_NewRef(func);
+    pto->placeholder = phold;
 
-    nargs = PyTuple_GetSlice(args, 1, PY_SSIZE_T_MAX);
-    if (nargs == NULL) {
+    new_args = PyTuple_GetSlice(args, 1, new_nargs + 1);
+    if (new_args == NULL) {
         Py_DECREF(pto);
         return NULL;
     }
-    if (pargs == NULL) {
-        pto->args = nargs;
+
+    /* Count placeholders */
+    Py_ssize_t phcount = 0;
+    for (Py_ssize_t i = 0; i < new_nargs - 1; i++) {
+        if (PyTuple_GET_ITEM(new_args, i) == phold) {
+            phcount++;
+        }
+    }
+    /* merge args with args of `func` which is `partial` */
+    if (pto_phcount > 0 && new_nargs > 0) {
+        Py_ssize_t npargs = PyTuple_GET_SIZE(pto_args);
+        Py_ssize_t tot_nargs = npargs;
+        if (new_nargs > pto_phcount) {
+            tot_nargs += new_nargs - pto_phcount;
+        }
+        PyObject *item;
+        PyObject *tot_args = PyTuple_New(tot_nargs);
+        for (Py_ssize_t i = 0, j = 0; i < tot_nargs; i++) {
+            if (i < npargs) {
+                item = PyTuple_GET_ITEM(pto_args, i);
+                if (j < new_nargs && item == phold) {
+                    item = PyTuple_GET_ITEM(new_args, j);
+                    j++;
+                    pto_phcount--;
+                }
+            }
+            else {
+                item = PyTuple_GET_ITEM(new_args, j);
+                j++;
+            }
+            Py_INCREF(item);
+            PyTuple_SET_ITEM(tot_args, i, item);
+        }
+        pto->args = tot_args;
+        pto->phcount = pto_phcount + phcount;
+        Py_DECREF(new_args);
+    }
+    else if (pto_args == NULL) {
+        pto->args = new_args;
+        pto->phcount = phcount;
     }
     else {
-        pto->args = PySequence_Concat(pargs, nargs);
-        Py_DECREF(nargs);
+        pto->args = PySequence_Concat(pto_args, new_args);
+        pto->phcount = pto_phcount + phcount;
+        Py_DECREF(new_args);
         if (pto->args == NULL) {
             Py_DECREF(pto);
             return NULL;
@@ -133,7 +261,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw)
         assert(PyTuple_Check(pto->args));
     }
 
-    if (pkw == NULL || PyDict_GET_SIZE(pkw) == 0) {
+    if (pto_kw == NULL || PyDict_GET_SIZE(pto_kw) == 0) {
         if (kw == NULL) {
             pto->kw = PyDict_New();
         }
@@ -145,7 +273,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw)
         }
     }
     else {
-        pto->kw = PyDict_Copy(pkw);
+        pto->kw = PyDict_Copy(pto_kw);
         if (kw != NULL && pto->kw != NULL) {
             if (PyDict_Merge(pto->kw, kw, 1) != 0) {
                 Py_DECREF(pto);
@@ -225,23 +353,30 @@ partial_vectorcall(partialobject *pto, PyObject *const *args,
                    size_t nargsf, PyObject *kwnames)
 {
     PyThreadState *tstate = _PyThreadState_GET();
+    Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
 
     /* pto->kw is mutable, so need to check every time */
     if (PyDict_GET_SIZE(pto->kw)) {
         return partial_vectorcall_fallback(tstate, pto, args, nargsf, kwnames);
     }
+    Py_ssize_t pto_phcount = pto->phcount;
+    if (nargs < pto_phcount) {
+        PyErr_Format(PyExc_TypeError,
+                     "missing positional arguments in 'partial' call; "
+                     "expected at least %zd, got %zd", pto_phcount, nargs);
+        return NULL;
+    }
 
-    Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
-    Py_ssize_t nargs_total = nargs;
+    Py_ssize_t nargskw = nargs;
     if (kwnames != NULL) {
-        nargs_total += PyTuple_GET_SIZE(kwnames);
+        nargskw += PyTuple_GET_SIZE(kwnames);
     }
 
     PyObject **pto_args = _PyTuple_ITEMS(pto->args);
     Py_ssize_t pto_nargs = PyTuple_GET_SIZE(pto->args);
 
     /* Fast path if we're called without arguments */
-    if (nargs_total == 0) {
+    if (nargskw == 0) {
         return _PyObject_VectorcallTstate(tstate, pto->fn,
                                           pto_args, pto_nargs, NULL);
     }
@@ -258,29 +393,47 @@ partial_vectorcall(partialobject *pto, PyObject *const *args,
         return ret;
     }
 
-    Py_ssize_t newnargs_total = pto_nargs + nargs_total;
-
     PyObject *small_stack[_PY_FASTCALL_SMALL_STACK];
-    PyObject *ret;
     PyObject **stack;
 
-    if (newnargs_total <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) {
+    Py_ssize_t tot_nargskw = pto_nargs + nargskw - pto_phcount;
+    if (tot_nargskw <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) {
         stack = small_stack;
     }
     else {
-        stack = PyMem_Malloc(newnargs_total * sizeof(PyObject *));
+        stack = PyMem_Malloc(tot_nargskw * sizeof(PyObject *));
         if (stack == NULL) {
             PyErr_NoMemory();
             return NULL;
         }
     }
 
-    /* Copy to new stack, using borrowed references */
-    memcpy(stack, pto_args, pto_nargs * sizeof(PyObject*));
-    memcpy(stack + pto_nargs, args, nargs_total * sizeof(PyObject*));
-
-    ret = _PyObject_VectorcallTstate(tstate, pto->fn,
-                                     stack, pto_nargs + nargs, kwnames);
+    Py_ssize_t tot_nargs;
+    if (pto_phcount) {
+        tot_nargs = pto_nargs + nargs - pto_phcount;
+        Py_ssize_t j = 0;       // New args index
+        for (Py_ssize_t i = 0; i < pto_nargs; i++) {
+            if (pto_args[i] == pto->placeholder) {
+                stack[i] = args[j];
+                j += 1;
+            }
+            else {
+                stack[i] = pto_args[i];
+            }
+        }
+        assert(j == pto_phcount);
+        if (nargskw > pto_phcount) {
+            memcpy(stack + pto_nargs, args + j, (nargskw - j) * sizeof(PyObject*));
+        }
+    }
+    else {
+        tot_nargs = pto_nargs + nargs;
+        /* Copy to new stack, using borrowed references */
+        memcpy(stack, pto_args, pto_nargs * sizeof(PyObject*));
+        memcpy(stack + pto_nargs, args, nargskw * sizeof(PyObject*));
+    }
+    PyObject *ret = _PyObject_VectorcallTstate(tstate, pto->fn,
+                                               stack, tot_nargs, kwnames);
     if (stack != small_stack) {
         PyMem_Free(stack);
     }
@@ -312,40 +465,81 @@ partial_call(partialobject *pto, PyObject *args, PyObject *kwargs)
     assert(PyTuple_Check(pto->args));
     assert(PyDict_Check(pto->kw));
 
+    Py_ssize_t nargs = PyTuple_GET_SIZE(args);
+    Py_ssize_t pto_phcount = pto->phcount;
+    if (nargs < pto_phcount) {
+        PyErr_Format(PyExc_TypeError,
+                     "missing positional arguments in 'partial' call; "
+                     "expected at least %zd, got %zd", pto_phcount, nargs);
+        return NULL;
+    }
+
     /* Merge keywords */
-    PyObject *kwargs2;
+    PyObject *tot_kw;
     if (PyDict_GET_SIZE(pto->kw) == 0) {
         /* kwargs can be NULL */
-        kwargs2 = Py_XNewRef(kwargs);
+        tot_kw = Py_XNewRef(kwargs);
     }
     else {
         /* bpo-27840, bpo-29318: dictionary of keyword parameters must be
            copied, because a function using "**kwargs" can modify the
            dictionary. */
-        kwargs2 = PyDict_Copy(pto->kw);
-        if (kwargs2 == NULL) {
+        tot_kw = PyDict_Copy(pto->kw);
+        if (tot_kw == NULL) {
             return NULL;
         }
 
         if (kwargs != NULL) {
-            if (PyDict_Merge(kwargs2, kwargs, 1) != 0) {
-                Py_DECREF(kwargs2);
+            if (PyDict_Merge(tot_kw, kwargs, 1) != 0) {
+                Py_DECREF(tot_kw);
                 return NULL;
             }
         }
     }
 
     /* Merge positional arguments */
-    /* Note: tupleconcat() is optimized for empty tuples */
-    PyObject *args2 = PySequence_Concat(pto->args, args);
-    if (args2 == NULL) {
-        Py_XDECREF(kwargs2);
-        return NULL;
+    PyObject *tot_args;
+    if (pto_phcount) {
+        Py_ssize_t pto_nargs = PyTuple_GET_SIZE(pto->args);
+        Py_ssize_t tot_nargs = pto_nargs + nargs - pto_phcount;
+        assert(tot_nargs >= 0);
+        tot_args = PyTuple_New(tot_nargs);
+        if (tot_args == NULL) {
+            Py_XDECREF(tot_kw);
+            return NULL;
+        }
+        PyObject *pto_args = pto->args;
+        PyObject *item;
+        Py_ssize_t j = 0;   // New args index
+        for (Py_ssize_t i = 0; i < pto_nargs; i++) {
+            item = PyTuple_GET_ITEM(pto_args, i);
+            if (item == pto->placeholder) {
+                item = PyTuple_GET_ITEM(args, j);
+                j += 1;
+            }
+            Py_INCREF(item);
+            PyTuple_SET_ITEM(tot_args, i, item);
+        }
+        assert(j == pto_phcount);
+        for (Py_ssize_t i = pto_nargs; i < tot_nargs; i++) {
+            item = PyTuple_GET_ITEM(args, j);
+            Py_INCREF(item);
+            PyTuple_SET_ITEM(tot_args, i, item);
+            j += 1;
+        }
+    }
+    else {
+        /* Note: tupleconcat() is optimized for empty tuples */
+        tot_args = PySequence_Concat(pto->args, args);
+        if (tot_args == NULL) {
+            Py_XDECREF(tot_kw);
+            return NULL;
+        }
     }
 
-    PyObject *res = PyObject_Call(pto->fn, args2, kwargs2);
-    Py_DECREF(args2);
-    Py_XDECREF(kwargs2);
+    PyObject *res = PyObject_Call(pto->fn, tot_args, tot_kw);
+    Py_DECREF(tot_args);
+    Py_XDECREF(tot_kw);
     return res;
 }
 
@@ -461,8 +655,11 @@ partial_setstate(partialobject *pto, PyObject *state)
 {
     PyObject *fn, *fnargs, *kw, *dict;
 
-    if (!PyTuple_Check(state) ||
-        !PyArg_ParseTuple(state, "OOOO", &fn, &fnargs, &kw, &dict) ||
+    if (!PyTuple_Check(state)) {
+        PyErr_SetString(PyExc_TypeError, "invalid partial state");
+        return NULL;
+    }
+    if (!PyArg_ParseTuple(state, "OOOO", &fn, &fnargs, &kw, &dict) ||
         !PyCallable_Check(fn) ||
         !PyTuple_Check(fnargs) ||
         (kw != Py_None && !PyDict_Check(kw)))
@@ -471,6 +668,20 @@ partial_setstate(partialobject *pto, PyObject *state)
         return NULL;
     }
 
+    Py_ssize_t nargs = PyTuple_GET_SIZE(fnargs);
+    if (nargs && PyTuple_GET_ITEM(fnargs, nargs - 1) == pto->placeholder) {
+        PyErr_SetString(PyExc_TypeError,
+                        "trailing Placeholders are not allowed");
+        return NULL;
+    }
+    /* Count placeholders */
+    Py_ssize_t phcount = 0;
+    for (Py_ssize_t i = 0; i < nargs - 1; i++) {
+        if (PyTuple_GET_ITEM(fnargs, i) == pto->placeholder) {
+            phcount++;
+        }
+    }
+
     if(!PyTuple_CheckExact(fnargs))
         fnargs = PySequence_Tuple(fnargs);
     else
@@ -493,10 +704,10 @@ partial_setstate(partialobject *pto, PyObject *state)
         dict = NULL;
     else
         Py_INCREF(dict);
-
     Py_SETREF(pto->fn, Py_NewRef(fn));
     Py_SETREF(pto->args, fnargs);
     Py_SETREF(pto->kw, kw);
+    pto->phcount = phcount;
     Py_XSETREF(pto->dict, dict);
     partial_setvectorcall(pto);
     Py_RETURN_NONE;
@@ -1498,6 +1709,21 @@ _functools_exec(PyObject *module)
         return -1;
     }
 
+    state->placeholder_type = (PyTypeObject *)PyType_FromModuleAndSpec(module,
+        &placeholder_type_spec, NULL);
+    if (state->placeholder_type == NULL) {
+        return -1;
+    }
+    if (PyModule_AddType(module, state->placeholder_type) < 0) {
+        return -1;
+    }
+    state->placeholder = PyObject_CallNoArgs((PyObject *)state->placeholder_type);
+    if (state->placeholder == NULL) {
+        return -1;
+    }
+    if (PyModule_AddObject(module, "Placeholder", state->placeholder) < 0) {
+        return -1;
+    }
     state->partial_type = (PyTypeObject *)PyType_FromModuleAndSpec(module,
         &partial_type_spec, NULL);
     if (state->partial_type == NULL) {
@@ -1542,6 +1768,7 @@ _functools_traverse(PyObject *module, visitproc visit, void *arg)
 {
     _functools_state *state = get_functools_state(module);
     Py_VISIT(state->kwd_mark);
+    Py_VISIT(state->placeholder_type);
     Py_VISIT(state->partial_type);
     Py_VISIT(state->keyobject_type);
     Py_VISIT(state->lru_list_elem_type);
@@ -1553,6 +1780,7 @@ _functools_clear(PyObject *module)
 {
     _functools_state *state = get_functools_state(module);
     Py_CLEAR(state->kwd_mark);
+    Py_CLEAR(state->placeholder_type);
     Py_CLEAR(state->partial_type);
     Py_CLEAR(state->keyobject_type);
     Py_CLEAR(state->lru_list_elem_type);