gh-93274: Expose receiving vectorcall in the Limited API (GH-95717)

Author: encukou

Doc/data/stable_abi.dat

@@ -426,14 +426,22 @@ New Features
   an additional metaclass argument.
   (Contributed by Wenzel Jakob in :gh:`93012`.)
 
-* (XXX: this should be combined with :gh:`93274` when that is done)
+* API for creating objects that can be called using
+  :ref:`the vectorcall protocol <vectorcall>` was added to the
+  :ref:`Limited API <stable>`:
+
+  * :const:`Py_TPFLAGS_HAVE_VECTORCALL`
+  * :c:func:`PyVectorcall_NARGS`
+  * :c:func:`PyVectorcall_Call`
+  * :c:type:`vectorcallfunc`
+
   The :const:`Py_TPFLAGS_HAVE_VECTORCALL` flag is now removed from a class
   when the class's :py:meth:`~object.__call__` method is reassigned.
   This makes vectorcall safe to use with mutable types (i.e. heap types
   without the :const:`immutable <Py_TPFLAGS_IMMUTABLETYPE>` flag).
   Mutable types that do not override :c:member:`~PyTypeObject.tp_call` now
-  inherit the :const:`Py_TPFLAGS_HAVE_VECTORCALL` flag.
-  (Contributed by Petr Viktorin in :gh:`93012`.)
+  inherit the ``Py_TPFLAGS_HAVE_VECTORCALL`` flag.
+  (Contributed by Petr Viktorin in :gh:`93274`.)
 
 Porting to Python 3.12
 ----------------------

Doc/whatsnew/3.12.rst

@@ -228,6 +228,16 @@ PyAPI_FUNC(PyObject *) PyObject_CallMethodObjArgs(
     PyObject *name,
     ...);
 
+/* Given a vectorcall nargsf argument, return the actual number of arguments.
+ * (For use outside the limited API, this is re-defined as a static inline
+ * function in cpython/abstract.h)
+ */
+PyAPI_FUNC(Py_ssize_t) PyVectorcall_NARGS(size_t nargsf);
+
+/* Call "callable" (which must support vectorcall) with positional arguments
+   "tuple" and keyword arguments "dict". "dict" may also be NULL */
+PyAPI_FUNC(PyObject *) PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *dict);
+
 
 /* Implemented elsewhere:
 

Include/abstract.h

@@ -53,8 +53,12 @@ PyAPI_FUNC(PyObject *) _PyObject_MakeTpCall(
 #define PY_VECTORCALL_ARGUMENTS_OFFSET \
     (_Py_STATIC_CAST(size_t, 1) << (8 * sizeof(size_t) - 1))
 
+// PyVectorcall_NARGS() is exported as a function for the stable ABI.
+// Here (when we are not using the stable ABI), the name is overridden to
+// call a static inline function for best performance.
+#define PyVectorcall_NARGS(n) _PyVectorcall_NARGS(n)
 static inline Py_ssize_t
-PyVectorcall_NARGS(size_t n)
+_PyVectorcall_NARGS(size_t n)
 {
     return n & ~PY_VECTORCALL_ARGUMENTS_OFFSET;
 }
@@ -84,10 +88,6 @@ PyAPI_FUNC(PyObject *) PyObject_VectorcallDict(
     size_t nargsf,
     PyObject *kwargs);
 
-/* Call "callable" (which must support vectorcall) with positional arguments
-   "tuple" and keyword arguments "dict". "dict" may also be NULL */
-PyAPI_FUNC(PyObject *) PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *dict);
-
 // Same as PyObject_Vectorcall(), except without keyword arguments
 PyAPI_FUNC(PyObject *) _PyObject_FastCall(
     PyObject *func,

Include/cpython/abstract.h

@@ -54,9 +54,6 @@ typedef struct _Py_Identifier {
 typedef int (*getbufferproc)(PyObject *, Py_buffer *, int);
 typedef void (*releasebufferproc)(PyObject *, Py_buffer *);
 
-typedef PyObject *(*vectorcallfunc)(PyObject *callable, PyObject *const *args,
-                                    size_t nargsf, PyObject *kwnames);
-
 
 typedef struct {
     /* Number implementations must check *both*

Include/cpython/object.h

@@ -228,6 +228,11 @@ typedef int (*initproc)(PyObject *, PyObject *, PyObject *);
 typedef PyObject *(*newfunc)(PyTypeObject *, PyObject *, PyObject *);
 typedef PyObject *(*allocfunc)(PyTypeObject *, Py_ssize_t);
 
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030c0000 // 3.12
+typedef PyObject *(*vectorcallfunc)(PyObject *callable, PyObject *const *args,
+                                    size_t nargsf, PyObject *kwnames);
+#endif
+
 typedef struct{
     int slot;    /* slot id, see below */
     void *pfunc; /* function pointer */
@@ -381,11 +386,13 @@ given type object has a specified feature.
 #define Py_TPFLAGS_BASETYPE (1UL << 10)
 
 /* Set if the type implements the vectorcall protocol (PEP 590) */
-#ifndef Py_LIMITED_API
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030C0000
 #define Py_TPFLAGS_HAVE_VECTORCALL (1UL << 11)
+#ifndef Py_LIMITED_API
 // Backwards compatibility alias for API that was provisional in Python 3.8
 #define _Py_TPFLAGS_HAVE_VECTORCALL Py_TPFLAGS_HAVE_VECTORCALL
 #endif
+#endif
 
 /* Set if the type is 'ready' -- fully initialized */
 #define Py_TPFLAGS_READY (1UL << 12)

Include/object.h

@@ -759,6 +759,11 @@ def __call__(self, *args):
                 self.assertEqual(expected, meth(*args1, **kwargs))
                 self.assertEqual(expected, wrapped(*args, **kwargs))
 
+    def test_vectorcall_limited(self):
+        from _testcapi import pyobject_vectorcall
+        obj = _testcapi.LimitedVectorCallClass()
+        self.assertEqual(pyobject_vectorcall(obj, (), ()), "vectorcall called")
+
 
 class A:
     def method_two_args(self, x, y):

Lib/test/test_call.py

@@ -0,0 +1,3 @@
+API for implementing vectorcall (:c:data:`Py_TPFLAGS_HAVE_VECTORCALL`,
+:c:func:`PyVectorcall_NARGS` and :c:func:`PyVectorcall_Call`) was added to
+the limited API and stable ABI.

Lib/test/test_stable_abi_ctypes.py

@@ -2275,5 +2275,14 @@
     added = '3.11'
 [function.PyErr_SetHandledException]
     added = '3.11'
+
 [function.PyType_FromMetaclass]
     added = '3.12'
+[const.Py_TPFLAGS_HAVE_VECTORCALL]
+    added = '3.12'
+[function.PyVectorcall_NARGS]
+    added = '3.12'
+[function.PyVectorcall_Call]
+    added = '3.12'
+[typedef.vectorcallfunc]
+    added = '3.12'

Misc/NEWS.d/next/C API/2022-08-01-16-21-39.gh-issue-93274.QoDHEu.rst

@@ -169,7 +169,7 @@
 @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c
 @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c
 @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c
-@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c
+@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c
 
 # Some testing modules MUST be built as shared libraries.
 *shared*

Misc/stable_abi.toml

@@ -1,4 +1,5 @@
 #include "Python.h"
 
 int _PyTestCapi_Init_Vectorcall(PyObject *module);
+int _PyTestCapi_Init_VectorcallLimited(PyObject *module);
 int _PyTestCapi_Init_Heaptype(PyObject *module);

Modules/Setup.stdlib.in

@@ -0,0 +1,77 @@
+#define Py_LIMITED_API 0x030c0000 // 3.12
+#include "parts.h"
+#include "structmember.h"         // PyMemberDef
+
+/* Test Vectorcall in the limited API */
+
+static PyObject *
+LimitedVectorCallClass_tpcall(PyObject *self, PyObject *args, PyObject *kwargs) {
+    return PyUnicode_FromString("tp_call called");
+}
+
+static PyObject *
+LimitedVectorCallClass_vectorcall(PyObject *callable,
+                            PyObject *const *args,
+                            size_t nargsf,
+                            PyObject *kwnames) {
+    return PyUnicode_FromString("vectorcall called");
+}
+
+static PyObject *
+LimitedVectorCallClass_new(PyTypeObject *tp, PyTypeObject *a, PyTypeObject *kw)
+{
+    PyObject *self = ((allocfunc)PyType_GetSlot(tp, Py_tp_alloc))(tp, 0);
+    if (!self) {
+        return NULL;
+    }
+    *(vectorcallfunc*)((char*)self + sizeof(PyObject)) = (
+        LimitedVectorCallClass_vectorcall);
+    return self;
+}
+
+static PyMemberDef LimitedVectorCallClass_members[] = {
+    {"__vectorcalloffset__", T_PYSSIZET, sizeof(PyObject), READONLY},
+    {NULL}
+};
+
+static PyType_Slot LimitedVectorallClass_slots[] = {
+    {Py_tp_new, LimitedVectorCallClass_new},
+    {Py_tp_call, LimitedVectorCallClass_tpcall},
+    {Py_tp_members, LimitedVectorCallClass_members},
+    {0},
+};
+
+static PyType_Spec LimitedVectorCallClass_spec = {
+    .name = "_testcapi.LimitedVectorCallClass",
+    .basicsize = (int)(sizeof(PyObject) + sizeof(vectorcallfunc)),
+    .flags = Py_TPFLAGS_DEFAULT
+        | Py_TPFLAGS_HAVE_VECTORCALL
+        | Py_TPFLAGS_BASETYPE,
+    .slots = LimitedVectorallClass_slots,
+};
+
+static PyMethodDef TestMethods[] = {
+    /* Add module methods here.
+     * (Empty list left here as template/example, since using
+     * PyModule_AddFunctions isn't very common.)
+     */
+    {NULL},
+};
+
+int
+_PyTestCapi_Init_VectorcallLimited(PyObject *m) {
+    if (PyModule_AddFunctions(m, TestMethods) < 0) {
+        return -1;
+    }
+
+    PyObject *LimitedVectorCallClass = PyType_FromModuleAndSpec(
+        m, &LimitedVectorCallClass_spec, NULL);
+    if (!LimitedVectorCallClass) {
+        return -1;
+    }
+    if (PyModule_AddType(m, (PyTypeObject *)LimitedVectorCallClass) < 0) {
+        return -1;
+    }
+
+    return 0;
+}

Modules/_testcapi/parts.h

@@ -6865,6 +6865,9 @@ PyInit__testcapi(void)
     if (_PyTestCapi_Init_Vectorcall(m) < 0) {
         return NULL;
     }
+    if (_PyTestCapi_Init_VectorcallLimited(m) < 0) {
+        return NULL;
+    }
     if (_PyTestCapi_Init_Heaptype(m) < 0) {
         return NULL;
     }

Modules/_testcapi/vectorcall_limited.c

@@ -1047,3 +1047,11 @@ _PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs,
     PyMem_Free((PyObject **)stack - 1);
     Py_DECREF(kwnames);
 }
+
+// Export for the stable ABI
+#undef PyVectorcall_NARGS
+Py_ssize_t
+PyVectorcall_NARGS(size_t n)
+{
+    return _PyVectorcall_NARGS(n);
+}

Modules/_testcapimodule.c

@@ -95,6 +95,7 @@
   <ItemGroup>
     <ClCompile Include="..\Modules\_testcapimodule.c" />
     <ClCompile Include="..\Modules\_testcapi\vectorcall.c" />
+    <ClCompile Include="..\Modules\_testcapi\vectorcall_limited.c" />
     <ClCompile Include="..\Modules\_testcapi\heaptype.c" />
   </ItemGroup>
   <ItemGroup>

Objects/call.c

@@ -15,6 +15,9 @@
     <ClCompile Include="..\Modules\_testcapi\vectorcall.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\Modules\_testcapi\vectorcall_limited.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\Modules\_testcapi\heaptype.c">
       <Filter>Source Files</Filter>
     </ClCompile>