gh-105387: Limited C API implements Py_INCREF() as func by vstinner · Pull Request #105388 · python/cpython (original) (raw)

What happens if I build with 3.12 but set Py_LIMITED_API to 3.11? or to just 1? It looks like I will get the 3.12 non-stable ABI version of Py_INCREF, rather than the 3.11 version of it.

Limited C API: Python 3.11 (inline)

You can look at xxlimited (Modules/xxlimited.c) which builds a C extension for the limited C API of Python 3.11 (#define Py_LIMITED_API 0x030b0000).

Preprocessor output (gcc -E):

static inline attribute((always_inline)) void Py_INCREF(PyObject *op) { uint32_t cur_refcnt = op->ob_refcnt_split[0]; uint32_t new_refcnt = cur_refcnt + 1; if (new_refcnt == 0) { return; } op->ob_refcnt_split[0] = new_refcnt; ((void)0); }

Extract of Xxo_getattro() x86-64 assembly code:

# C code
        PyObject *v = PyDict_GetItemWithError(self->x_attr, name);
        if (v != NULL) {
            return Py_NewRef(v);

# Assembly code
   # v = PyDict_GetItemWithError()
   0x00007ffff7c7f533 <+19>:	call   0x7ffff7c7f1b0 <PyDict_GetItemWithError@plt>
   # rbp = v
   0x00007ffff7c7f538 <+24>:	mov    rbp,rax

   # if (v == NULL) ...
   0x00007ffff7c7f53b <+27>:	test   rax,rax
   0x00007ffff7c7f53e <+30>:	je     0x7ffff7c7f558 <Xxo_getattro+56>

    # Py_NewRef(v) => Py_INCREF(v)
   0x00007ffff7c7f540 <+32>:	mov    eax,DWORD PTR [rax]
   0x00007ffff7c7f542 <+34>:	add    eax,0x1
   0x00007ffff7c7f545 <+37>:	je     0x7ffff7c7f54a <Xxo_getattro+42>  # immutable refcnt?
   0x00007ffff7c7f547 <+39>:	mov    DWORD PTR [rbp+0x0],eax

   # return v => return rbp
   0x00007ffff7c7f54a <+42>:	mov    rax,rbp
   0x00007ffff7c7f54d <+45>:	pop    rbx
   0x00007ffff7c7f54e <+46>:	pop    rbp
   0x00007ffff7c7f54f <+47>:	pop    r12
   0x00007ffff7c7f551 <+49>:	ret

=> with the limited C API version 3.11, Py_INCREF() is inlined code (refcnt+1 and test for immutable refcnt) at the ABI level.

Limited C API: Python 3.12 (function call)

I modified xxlimited (Modules/xxlimited.c) to build the C extension for the limited C API of Python 3.12 (#define Py_LIMITED_API 0x030c0000).

Preprocessor output (gcc -E):

static inline attribute((always_inline)) void Py_INCREF(PyObject *op) { _Py_IncRef(op); }

Extract of Xxo_getattro() x86-64 assembly code:

# C code
        PyObject *v = PyDict_GetItemWithError(self->x_attr, name);
        if (v != NULL) {
            return Py_NewRef(v);

# Assembly code
   # v = PyDict_GetItemWithError()
   0x00007ffff7c7f4e3 <+19>:	call   0x7ffff7c7f1c0 <PyDict_GetItemWithError@plt>
   # r12 = v
   0x00007ffff7c7f4e8 <+24>:	mov    r12,rax

   # if (v == NULL) ...
   0x00007ffff7c7f4eb <+27>:	test   rax,rax
   0x00007ffff7c7f4ee <+30>:	je     0x7ffff7c7f500 <Xxo_getattro+48>

   # Py_INCREF(v) => _Py_IncRef(v)
   0x00007ffff7c7f4f0 <+32>:	mov    rdi,rax
   0x00007ffff7c7f4f3 <+35>:	call   0x7ffff7c7f030 <_Py_IncRef@plt>

   # return v => return r12
   0x00007ffff7c7f4f8 <+40>:	mov    rax,r12
   0x00007ffff7c7f4fb <+43>:	pop    rbx
   0x00007ffff7c7f4fc <+44>:	pop    rbp
   0x00007ffff7c7f4fd <+45>:	pop    r12
   0x00007ffff7c7f4ff <+47>:	ret

=> with the limited C API version 3.12, Py_INCREF() becomes a function call to _Py_IncRef() at the ABI level.

Py_LIMITED_API=1 (inline)

I modified Modules/xxlimited.c to use #define Py_LIMITED_API 1.

Preprocessor output (gcc -E):

static inline attribute((always_inline)) void Py_INCREF(PyObject *op) { uint32_t cur_refcnt = op->ob_refcnt_split[0]; uint32_t new_refcnt = cur_refcnt + 1; if (new_refcnt == 0) { return; } op->ob_refcnt_split[0] = new_refcnt; ((void)0); }

=> With #define Py_LIMITED_API 1, I get the same Py_INCREF() inlined code than limited C API version 3.11 (#define Py_LIMITED_API 0x030b0000) at the ABI level.

#define Py_LIMITED_API 1 asks for the limited C API version 3.2: maximum backward compatibility.

The trick is to convert Py_LIMITED_API to an integer with Py_LIMITED_API+0 and uses it to compare the version:

#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG))

It's a common pattern in the limited C API: look for Py_LIMITED_API usage in Include/*.h files.