From d812710190e9b22314fcafbd755bec4a1a247f95 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 19 Apr 2022 14:56:54 +0200 Subject: [PATCH 1/2] gh-89653: PEP 670: Functions don't cast pointers In the limited C API version 3.11 and newer, the following functions no longer cast their object pointer argument with _PyObject_CAST() or _PyObject_CAST_CONST(): * Py_REFCNT(), Py_TYPE(), Py_SIZE() * Py_SET_REFCNT(), Py_SET_TYPE(), Py_SET_SIZE() * Py_IS_TYPE() * Py_INCREF(), Py_DECREF() * Py_XINCREF(), Py_XDECREF() * Py_NewRef(), Py_XNewRef() * PyObject_TypeCheck() * PyType_Check() * PyType_CheckExact() Split Py_DECREF() implementation in 3 versions to make the code more readable. Update the xxlimited.c extension, which uses the limited C API version 3.11, to pass PyObject* to these functions. --- Include/object.h | 101 +++++++++++++++++++++++++++++--------------- Modules/xxlimited.c | 10 +++-- 2 files changed, 72 insertions(+), 39 deletions(-) diff --git a/Include/object.h b/Include/object.h index 8a45ddbf6057ed..5622305f49f78f 100644 --- a/Include/object.h +++ b/Include/object.h @@ -123,44 +123,58 @@ PyAPI_FUNC(int) Py_Is(PyObject *x, PyObject *y); static inline Py_ssize_t Py_REFCNT(PyObject *ob) { return ob->ob_refcnt; } -#define Py_REFCNT(ob) Py_REFCNT(_PyObject_CAST(ob)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_REFCNT(ob) Py_REFCNT(_PyObject_CAST(ob)) +#endif // bpo-39573: The Py_SET_TYPE() function must be used to set an object type. static inline PyTypeObject* Py_TYPE(PyObject *ob) { return ob->ob_type; } -#define Py_TYPE(ob) Py_TYPE(_PyObject_CAST(ob)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_TYPE(ob) Py_TYPE(_PyObject_CAST(ob)) +#endif // bpo-39573: The Py_SET_SIZE() function must be used to set an object size. static inline Py_ssize_t Py_SIZE(PyVarObject *ob) { return ob->ob_size; } -#define Py_SIZE(ob) Py_SIZE(_PyVarObject_CAST(ob)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_SIZE(ob) Py_SIZE(_PyVarObject_CAST(ob)) +#endif static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { return Py_TYPE(ob) == type; } -#define Py_IS_TYPE(ob, type) Py_IS_TYPE(_PyObject_CAST(ob), type) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_IS_TYPE(ob, type) Py_IS_TYPE(_PyObject_CAST(ob), type) +#endif static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { ob->ob_refcnt = refcnt; } -#define Py_SET_REFCNT(ob, refcnt) Py_SET_REFCNT(_PyObject_CAST(ob), refcnt) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_SET_REFCNT(ob, refcnt) Py_SET_REFCNT(_PyObject_CAST(ob), refcnt) +#endif static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { ob->ob_type = type; } -#define Py_SET_TYPE(ob, type) Py_SET_TYPE(_PyObject_CAST(ob), type) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_SET_TYPE(ob, type) Py_SET_TYPE(_PyObject_CAST(ob), type) +#endif static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { ob->ob_size = size; } -#define Py_SET_SIZE(ob, size) Py_SET_SIZE(_PyVarObject_CAST(ob), size) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_SET_SIZE(ob, size) Py_SET_SIZE(_PyVarObject_CAST(ob), size) +#endif /* @@ -247,7 +261,9 @@ PyAPI_FUNC(int) PyType_IsSubtype(PyTypeObject *, PyTypeObject *); static inline int PyObject_TypeCheck(PyObject *ob, PyTypeObject *type) { return Py_IS_TYPE(ob, type) || PyType_IsSubtype(Py_TYPE(ob), type); } -#define PyObject_TypeCheck(ob, type) PyObject_TypeCheck(_PyObject_CAST(ob), type) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define PyObject_TypeCheck(ob, type) PyObject_TypeCheck(_PyObject_CAST(ob), type) +#endif PyAPI_DATA(PyTypeObject) PyType_Type; /* built-in 'type' */ PyAPI_DATA(PyTypeObject) PyBaseObject_Type; /* built-in 'object' */ @@ -485,39 +501,43 @@ static inline void Py_INCREF(PyObject *op) op->ob_refcnt++; #endif } -#define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op)) - -static inline void Py_DECREF( -#if defined(Py_REF_DEBUG) && !(defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000) - const char *filename, int lineno, +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op)) #endif - PyObject *op) -{ + + #if defined(Py_REF_DEBUG) && defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000 - // Stable ABI for Python 3.10 built in debug mode. +// Stable ABI for limited C API version 3.10 of Python debug build +static inline void Py_DECREF(PyObject *op) { _Py_DecRef(op); -#else - // Non-limited C API and limited C API for Python 3.9 and older access - // directly PyObject.ob_refcnt. -#ifdef Py_REF_DEBUG +} +#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) + +#elif defined(Py_REF_DEBUG) +static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) +{ _Py_RefTotal--; -#endif if (--op->ob_refcnt != 0) { -#ifdef Py_REF_DEBUG if (op->ob_refcnt < 0) { _Py_NegativeRefcount(filename, lineno, op); } -#endif } else { _Py_Dealloc(op); } -#endif } -#if defined(Py_REF_DEBUG) && !(defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000) -# define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) +#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) + #else -# define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) +static inline void Py_DECREF(PyObject *op) +{ + // Non-limited C API and limited C API for Python 3.9 and older access + // directly PyObject.ob_refcnt. + if (--op->ob_refcnt == 0) { + _Py_Dealloc(op); + } +} +#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) #endif @@ -571,8 +591,9 @@ static inline void Py_XINCREF(PyObject *op) Py_INCREF(op); } } - -#define Py_XINCREF(op) Py_XINCREF(_PyObject_CAST(op)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_XINCREF(op) Py_XINCREF(_PyObject_CAST(op)) +#endif static inline void Py_XDECREF(PyObject *op) { @@ -580,8 +601,9 @@ static inline void Py_XDECREF(PyObject *op) Py_DECREF(op); } } - -#define Py_XDECREF(op) Py_XDECREF(_PyObject_CAST(op)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_XDECREF(op) Py_XDECREF(_PyObject_CAST(op)) +#endif // Create a new strong reference to an object: // increment the reference count of the object and return the object. @@ -605,8 +627,13 @@ static inline PyObject* _Py_XNewRef(PyObject *obj) // Py_NewRef() and Py_XNewRef() are exported as functions for the stable ABI. // Names overridden with macros by static inline functions for best // performances. -#define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj)) -#define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj)) +# define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj)) +#else +# define Py_NewRef(obj) _Py_NewRef(obj) +# define Py_XNewRef(obj) _Py_XNewRef(obj) +#endif /* @@ -749,14 +776,18 @@ PyType_HasFeature(PyTypeObject *type, unsigned long feature) static inline int PyType_Check(PyObject *op) { return PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_TYPE_SUBCLASS); } -#define PyType_Check(op) PyType_Check(_PyObject_CAST(op)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define PyType_Check(op) PyType_Check(_PyObject_CAST(op)) +#endif #define _PyType_CAST(op) (assert(PyType_Check(op)), (PyTypeObject*)(op)) static inline int PyType_CheckExact(PyObject *op) { return Py_IS_TYPE(op, &PyType_Type); } -#define PyType_CheckExact(op) PyType_CheckExact(_PyObject_CAST(op)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define PyType_CheckExact(op) PyType_CheckExact(_PyObject_CAST(op)) +#endif #ifdef __cplusplus } diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 16d1b8311c62cc..c36bf5ce8e1f76 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -110,12 +110,13 @@ newXxoObject(PyObject *module) /* Xxo finalization */ static int -Xxo_traverse(XxoObject *self, visitproc visit, void *arg) +Xxo_traverse(PyObject *self_obj, visitproc visit, void *arg) { // Visit the type - Py_VISIT(Py_TYPE(self)); + Py_VISIT(Py_TYPE(self_obj)); // Visit the attribute dict + XxoObject *self = (XxoObject *)self_obj; Py_VISIT(self->x_attr); return 0; } @@ -128,13 +129,14 @@ Xxo_clear(XxoObject *self) } static void -Xxo_finalize(XxoObject *self) +Xxo_finalize(PyObject *self_obj) { + XxoObject *self = (XxoObject *)self_obj; Py_CLEAR(self->x_attr); } static void -Xxo_dealloc(XxoObject *self) +Xxo_dealloc(PyObject *self) { Xxo_finalize(self); PyTypeObject *tp = Py_TYPE(self); From 3fe8226147ccd8d43348e1622292a5791d308649 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 21 Apr 2022 23:33:36 +0200 Subject: [PATCH 2/2] Update unicodeobject.h --- Include/cpython/unicodeobject.h | 64 ++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 8cc9885756037d..380b7629517c30 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -256,14 +256,18 @@ PyAPI_FUNC(int) _PyUnicode_CheckConsistency( static inline unsigned int PyUnicode_CHECK_INTERNED(PyObject *op) { return _PyASCIIObject_CAST(op)->state.interned; } -#define PyUnicode_CHECK_INTERNED(op) PyUnicode_CHECK_INTERNED(_PyObject_CAST(op)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define PyUnicode_CHECK_INTERNED(op) PyUnicode_CHECK_INTERNED(_PyObject_CAST(op)) +#endif /* Fast check to determine whether an object is ready. Equivalent to: PyUnicode_IS_COMPACT(op) || _PyUnicodeObject_CAST(op)->data.any */ static inline unsigned int PyUnicode_IS_READY(PyObject *op) { return _PyASCIIObject_CAST(op)->state.ready; } -#define PyUnicode_IS_READY(op) PyUnicode_IS_READY(_PyObject_CAST(op)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define PyUnicode_IS_READY(op) PyUnicode_IS_READY(_PyObject_CAST(op)) +#endif /* Return true if the string contains only ASCII characters, or 0 if not. The string may be compact (PyUnicode_IS_COMPACT_ASCII) or not, but must be @@ -272,21 +276,27 @@ static inline unsigned int PyUnicode_IS_ASCII(PyObject *op) { assert(PyUnicode_IS_READY(op)); return _PyASCIIObject_CAST(op)->state.ascii; } -#define PyUnicode_IS_ASCII(op) PyUnicode_IS_ASCII(_PyObject_CAST(op)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define PyUnicode_IS_ASCII(op) PyUnicode_IS_ASCII(_PyObject_CAST(op)) +#endif /* Return true if the string is compact or 0 if not. No type checks or Ready calls are performed. */ static inline unsigned int PyUnicode_IS_COMPACT(PyObject *op) { return _PyASCIIObject_CAST(op)->state.compact; } -#define PyUnicode_IS_COMPACT(op) PyUnicode_IS_COMPACT(_PyObject_CAST(op)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define PyUnicode_IS_COMPACT(op) PyUnicode_IS_COMPACT(_PyObject_CAST(op)) +#endif /* Return true if the string is a compact ASCII string (use PyASCIIObject structure), or 0 if not. No type checks or Ready calls are performed. */ static inline int PyUnicode_IS_COMPACT_ASCII(PyObject *op) { return (_PyASCIIObject_CAST(op)->state.ascii && PyUnicode_IS_COMPACT(op)); } -#define PyUnicode_IS_COMPACT_ASCII(op) PyUnicode_IS_COMPACT_ASCII(_PyObject_CAST(op)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define PyUnicode_IS_COMPACT_ASCII(op) PyUnicode_IS_COMPACT_ASCII(_PyObject_CAST(op)) +#endif enum PyUnicode_Kind { /* String contains only wstr byte characters. This is only possible @@ -325,7 +335,9 @@ static inline void* PyUnicode_DATA(PyObject *op) { } return _PyUnicode_NONCOMPACT_DATA(op); } -#define PyUnicode_DATA(op) PyUnicode_DATA(_PyObject_CAST(op)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define PyUnicode_DATA(op) PyUnicode_DATA(_PyObject_CAST(op)) +#endif /* Return pointers to the canonical representation cast to unsigned char, Py_UCS2, or Py_UCS4 for direct character access. @@ -343,7 +355,9 @@ static inline Py_ssize_t PyUnicode_GET_LENGTH(PyObject *op) { assert(PyUnicode_IS_READY(op)); return _PyASCIIObject_CAST(op)->length; } -#define PyUnicode_GET_LENGTH(op) PyUnicode_GET_LENGTH(_PyObject_CAST(op)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define PyUnicode_GET_LENGTH(op) PyUnicode_GET_LENGTH(_PyObject_CAST(op)) +#endif /* Write into the canonical representation, this function does not do any sanity checks and is intended for usage in loops. The caller should cache the @@ -399,8 +413,10 @@ static inline Py_UCS4 PyUnicode_READ_CHAR(PyObject *unicode, Py_ssize_t index) } return PyUnicode_4BYTE_DATA(unicode)[index]; } -#define PyUnicode_READ_CHAR(unicode, index) \ - PyUnicode_READ_CHAR(_PyObject_CAST(unicode), (index)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define PyUnicode_READ_CHAR(unicode, index) \ + PyUnicode_READ_CHAR(_PyObject_CAST(unicode), (index)) +#endif /* Return a maximum character value which is suitable for creating another string based on op. This is always an approximation but more efficient @@ -421,8 +437,10 @@ static inline Py_UCS4 PyUnicode_MAX_CHAR_VALUE(PyObject *op) } return 0x10ffffU; } -#define PyUnicode_MAX_CHAR_VALUE(op) \ - PyUnicode_MAX_CHAR_VALUE(_PyObject_CAST(op)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define PyUnicode_MAX_CHAR_VALUE(op) \ + PyUnicode_MAX_CHAR_VALUE(_PyObject_CAST(op)) +#endif /* === Public API ========================================================= */ @@ -458,7 +476,9 @@ static inline int PyUnicode_READY(PyObject *op) } return _PyUnicode_Ready(op); } -#define PyUnicode_READY(op) PyUnicode_READY(_PyObject_CAST(op)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define PyUnicode_READY(op) PyUnicode_READY(_PyObject_CAST(op)) +#endif /* Get a copy of a Unicode string. */ PyAPI_FUNC(PyObject*) _PyUnicode_Copy( @@ -599,7 +619,9 @@ static inline Py_ssize_t PyUnicode_WSTR_LENGTH(PyObject *op) return _PyCompactUnicodeObject_CAST(op)->wstr_length; } } -#define PyUnicode_WSTR_LENGTH(op) PyUnicode_WSTR_LENGTH(_PyObject_CAST(op)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define PyUnicode_WSTR_LENGTH(op) PyUnicode_WSTR_LENGTH(_PyObject_CAST(op)) +#endif /* Returns the deprecated Py_UNICODE representation's size in code units (this includes surrogate pairs as 2 units). @@ -618,14 +640,18 @@ static inline Py_ssize_t PyUnicode_GET_SIZE(PyObject *op) return PyUnicode_WSTR_LENGTH(op); _Py_COMP_DIAG_POP } -#define PyUnicode_GET_SIZE(op) PyUnicode_GET_SIZE(_PyObject_CAST(op)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define PyUnicode_GET_SIZE(op) PyUnicode_GET_SIZE(_PyObject_CAST(op)) +#endif /* Py_DEPRECATED(3.3) */ static inline Py_ssize_t PyUnicode_GET_DATA_SIZE(PyObject *op) { return PyUnicode_GET_SIZE(op) * Py_UNICODE_SIZE; } -#define PyUnicode_GET_DATA_SIZE(op) PyUnicode_GET_DATA_SIZE(_PyObject_CAST(op)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define PyUnicode_GET_DATA_SIZE(op) PyUnicode_GET_DATA_SIZE(_PyObject_CAST(op)) +#endif /* Alias for PyUnicode_AsUnicode(). This will create a wchar_t/Py_UNICODE representation on demand. Using this macro is very inefficient now, @@ -645,14 +671,18 @@ static inline Py_UNICODE* PyUnicode_AS_UNICODE(PyObject *op) return PyUnicode_AsUnicode(op); _Py_COMP_DIAG_POP } -#define PyUnicode_AS_UNICODE(op) PyUnicode_AS_UNICODE(_PyObject_CAST(op)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define PyUnicode_AS_UNICODE(op) PyUnicode_AS_UNICODE(_PyObject_CAST(op)) +#endif /* Py_DEPRECATED(3.3) */ static inline const char* PyUnicode_AS_DATA(PyObject *op) { return (const char *)PyUnicode_AS_UNICODE(op); } -#define PyUnicode_AS_DATA(op) PyUnicode_AS_DATA(_PyObject_CAST(op)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define PyUnicode_AS_DATA(op) PyUnicode_AS_DATA(_PyObject_CAST(op)) +#endif /* --- _PyUnicodeWriter API ----------------------------------------------- */