pythongh-76785: Improved Subinterpreters Compatibility with 3.12 (1/2) (pythongh-126704)

These changes makes it easier to backport the _interpreters, _interpqueues, and _interpchannels modules to Python 3.12.

This involves the following:

* rename several structs and typedefs
* add several typedefs
* stop using the PyThreadState.state field directly in parking_lot.c

Author: ericsnowcurrently

Include/internal/pycore_crossinterp.h

@@ -39,14 +39,14 @@ extern int _Py_CallInInterpreterAndRawFree(
 /* cross-interpreter data */
 /**************************/
 
-typedef struct _xid _PyXIData_t;
-typedef PyObject *(*xid_newobjectfunc)(_PyXIData_t *);
+typedef struct _xidata _PyXIData_t;
+typedef PyObject *(*xid_newobjfunc)(_PyXIData_t *);
 typedef void (*xid_freefunc)(void *);
 
 // _PyXIData_t is similar to Py_buffer as an effectively
 // opaque struct that holds data outside the object machinery.  This
 // is necessary to pass safely between interpreters in the same process.
-struct _xid {
+struct _xidata {
     // data is the cross-interpreter-safe derivation of a Python object
     // (see _PyObject_GetXIData).  It will be NULL if the
     // new_object func (below) encodes the data.
@@ -72,7 +72,7 @@ struct _xid {
     // interpreter given the data.  The resulting object (a new
     // reference) will be equivalent to the original object.  This field
     // is required.
-    xid_newobjectfunc new_object;
+    xid_newobjfunc new_object;
     // free is called when the data is released.  If it is NULL then
     // nothing will be done to free the data.  For some types this is
     // okay (e.g. bytes) and for those types this field should be set
@@ -117,11 +117,11 @@ PyAPI_FUNC(int) _PyXIData_ReleaseAndRawFree(_PyXIData_t *);
 PyAPI_FUNC(void) _PyXIData_Init(
         _PyXIData_t *data,
         PyInterpreterState *interp, void *shared, PyObject *obj,
-        xid_newobjectfunc new_object);
+        xid_newobjfunc new_object);
 PyAPI_FUNC(int) _PyXIData_InitWithSize(
         _PyXIData_t *,
         PyInterpreterState *interp, const size_t, PyObject *,
-        xid_newobjectfunc);
+        xid_newobjfunc);
 PyAPI_FUNC(void) _PyXIData_Clear( PyInterpreterState *, _PyXIData_t *);
 
 // Normally the Init* functions are sufficient.  The only time
@@ -155,12 +155,12 @@ PyAPI_FUNC(void) _PyXIData_Clear( PyInterpreterState *, _PyXIData_t *);
 /* runtime state & lifecycle */
 /*****************************/
 
-struct _xi_runtime_state {
+typedef struct {
     // builtin types
     _PyXIData_lookup_t data_lookup;
-};
+} _PyXI_global_state_t;
 
-struct _xi_state {
+typedef struct {
     // heap types
     _PyXIData_lookup_t data_lookup;
 
@@ -171,7 +171,7 @@ struct _xi_state {
         // heap types
         PyObject *PyExc_NotShareableError;
     } exceptions;
-};
+} _PyXI_state_t;
 
 extern PyStatus _PyXI_Init(PyInterpreterState *interp);
 extern void _PyXI_Fini(PyInterpreterState *interp);

Include/internal/pycore_crossinterp_data_registry.h

@@ -7,30 +7,30 @@
 // alternative would be to add a tp_* slot for a class's
 // xidatafunc. It would be simpler and more efficient.
 
-struct _xidregitem;
+struct _xid_regitem;
 
-struct _xidregitem {
-    struct _xidregitem *prev;
-    struct _xidregitem *next;
+typedef struct _xid_regitem {
+    struct _xid_regitem *prev;
+    struct _xid_regitem *next;
     /* This can be a dangling pointer, but only if weakref is set. */
     PyTypeObject *cls;
     /* This is NULL for builtin types. */
     PyObject *weakref;
     size_t refcount;
     xidatafunc getdata;
-};
+} _PyXIData_regitem_t;
 
-struct _xidregistry {
+typedef struct {
     int global;  /* builtin types or heap types */
     int initialized;
     PyMutex mutex;
-    struct _xidregitem *head;
-};
+    _PyXIData_regitem_t *head;
+} _PyXIData_registry_t;
 
 PyAPI_FUNC(int) _PyXIData_RegisterClass(PyTypeObject *, xidatafunc);
 PyAPI_FUNC(int) _PyXIData_UnregisterClass(PyTypeObject *);
 
 struct _xid_lookup_state {
     // XXX Remove this field once we have a tp_* slot.
-    struct _xidregistry registry;
+    _PyXIData_registry_t registry;
 };

Include/internal/pycore_interp.h

@@ -16,7 +16,7 @@ extern "C" {
 #include "pycore_code.h"          // struct callable_cache
 #include "pycore_codecs.h"        // struct codecs_state
 #include "pycore_context.h"       // struct _Py_context_state
-#include "pycore_crossinterp.h"   // struct _xidregistry
+#include "pycore_crossinterp.h"   // _PyXI_state_t
 #include "pycore_dict_state.h"    // struct _Py_dict_state
 #include "pycore_dtoa.h"          // struct _dtoa_state
 #include "pycore_exceptions.h"    // struct _Py_exc_state
@@ -205,7 +205,7 @@ struct _is {
     freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS];
 
     /* cross-interpreter data and utils */
-    struct _xi_state xi;
+    _PyXI_state_t xi;
 
 #ifdef HAVE_FORK
     PyObject *before_forkers;

Include/internal/pycore_pystate.h

@@ -141,6 +141,12 @@ _PyThreadState_GET(void)
 #endif
 }
 
+static inline int
+_PyThreadState_IsAttached(PyThreadState *tstate)
+{
+    return (_Py_atomic_load_int_relaxed(&tstate->state) == _Py_THREAD_ATTACHED);
+}
+
 // Attaches the current thread to the interpreter.
 //
 // This may block while acquiring the GIL (if the GIL is enabled) or while

Include/internal/pycore_runtime.h

@@ -11,7 +11,7 @@ extern "C" {
 #include "pycore_atexit.h"          // struct _atexit_runtime_state
 #include "pycore_audit.h"           // _Py_AuditHookEntry
 #include "pycore_ceval_state.h"     // struct _ceval_runtime_state
-#include "pycore_crossinterp.h"     // struct _xidregistry
+#include "pycore_crossinterp.h"     // _PyXI_global_state_t
 #include "pycore_debug_offsets.h"   // _Py_DebugOffsets
 #include "pycore_faulthandler.h"    // struct _faulthandler_runtime_state
 #include "pycore_floatobject.h"     // struct _Py_float_runtime_state
@@ -106,7 +106,7 @@ typedef struct pyruntimestate {
      tools. */
 
     /* cross-interpreter data and utils */
-    struct _xi_runtime_state xi;
+    _PyXI_global_state_t xi;
 
     struct _pymem_allocators allocators;
     struct _obmalloc_global_state obmalloc;

Modules/_interpchannelsmodule.c

@@ -63,7 +63,7 @@ _globals (static struct globals):
                             data (void *)
                             obj (PyObject *)
                             interpid (int64_t)
-                            new_object (xid_newobjectfunc)
+                            new_object (xid_newobjfunc)
                             free (xid_freefunc)
                     last (struct _channelitem *):
                         ...

Python/crossinterp.c

@@ -126,7 +126,7 @@ void
 _PyXIData_Init(_PyXIData_t *data,
                PyInterpreterState *interp,
                void *shared, PyObject *obj,
-               xid_newobjectfunc new_object)
+               xid_newobjfunc new_object)
 {
     assert(data != NULL);
     assert(new_object != NULL);
@@ -150,7 +150,7 @@ int
 _PyXIData_InitWithSize(_PyXIData_t *data,
                        PyInterpreterState *interp,
                        const size_t size, PyObject *obj,
-                       xid_newobjectfunc new_object)
+                       xid_newobjfunc new_object)
 {
     assert(size > 0);
     // For now we always free the shared data in the same interpreter
@@ -202,11 +202,9 @@ _check_xidata(PyThreadState *tstate, _PyXIData_t *data)
 }
 
 static inline void
-_set_xid_lookup_failure(PyInterpreterState *interp,
-                        PyObject *obj, const char *msg)
+_set_xid_lookup_failure(_PyXI_state_t *state, PyObject *obj, const char *msg)
 {
-    exceptions_t *state = &_PyInterpreterState_GetXIState(interp)->exceptions;
-    PyObject *exctype = state->PyExc_NotShareableError;
+    PyObject *exctype = state->exceptions.PyExc_NotShareableError;
     assert(exctype != NULL);
     if (msg != NULL) {
         assert(obj == NULL);
@@ -226,10 +224,11 @@ int
 _PyObject_CheckXIData(PyObject *obj)
 {
     PyInterpreterState *interp = PyInterpreterState_Get();
+    _PyXI_state_t *state = _PyXI_GET_STATE(interp);
     xidatafunc getdata = lookup_getdata(interp, obj);
     if (getdata == NULL) {
         if (!PyErr_Occurred()) {
-            _set_xid_lookup_failure(interp, obj, NULL);
+            _set_xid_lookup_failure(state, obj, NULL);
         }
         return -1;
     }
@@ -241,6 +240,7 @@ _PyObject_GetXIData(PyObject *obj, _PyXIData_t *data)
 {
     PyThreadState *tstate = PyThreadState_Get();
     PyInterpreterState *interp = tstate->interp;
+    _PyXI_state_t *state = _PyXI_GET_STATE(interp);
 
     // Reset data before re-populating.
     *data = (_PyXIData_t){0};
@@ -252,7 +252,7 @@ _PyObject_GetXIData(PyObject *obj, _PyXIData_t *data)
     if (getdata == NULL) {
         Py_DECREF(obj);
         if (!PyErr_Occurred()) {
-            _set_xid_lookup_failure(interp, obj, NULL);
+            _set_xid_lookup_failure(state, obj, NULL);
         }
         return -1;
     }
@@ -969,6 +969,7 @@ _PyXI_ClearExcInfo(_PyXI_excinfo *info)
 static int
 _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp)
 {
+    _PyXI_state_t *state;
     assert(!PyErr_Occurred());
     switch (code) {
     case _PyXI_ERR_NO_ERROR: _Py_FALLTHROUGH;
@@ -999,7 +1000,8 @@ _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp)
                         "failed to apply namespace to __main__");
         break;
     case _PyXI_ERR_NOT_SHAREABLE:
-        _set_xid_lookup_failure(interp, NULL, NULL);
+        state = _PyXI_GET_STATE(interp);
+        _set_xid_lookup_failure(state, NULL, NULL);
         break;
     default:
 #ifdef Py_DEBUG
@@ -1061,7 +1063,8 @@ _PyXI_ApplyError(_PyXI_error *error)
     }
     else if (error->code == _PyXI_ERR_NOT_SHAREABLE) {
         // Propagate the exception directly.
-        _set_xid_lookup_failure(error->interp, NULL, error->uncaught.msg);
+        _PyXI_state_t *state = _PyXI_GET_STATE(error->interp);
+        _set_xid_lookup_failure(state, NULL, error->uncaught.msg);
     }
     else {
         // Raise an exception corresponding to the code.
@@ -1606,9 +1609,9 @@ _propagate_not_shareable_error(_PyXI_session *session)
         return;
     }
     PyInterpreterState *interp = PyInterpreterState_Get();
-    exceptions_t *state = &_PyInterpreterState_GetXIState(interp)->exceptions;
-    assert(state->PyExc_NotShareableError != NULL);
-    if (PyErr_ExceptionMatches(state->PyExc_NotShareableError)) {
+    _PyXI_state_t *state = _PyXI_GET_STATE(interp);
+    assert(state->exceptions.PyExc_NotShareableError != NULL);
+    if (PyErr_ExceptionMatches(state->exceptions.PyExc_NotShareableError)) {
         // We want to propagate the exception directly.
         session->_error_override = _PyXI_ERR_NOT_SHAREABLE;
         session->error_override = &session->_error_override;
@@ -1779,11 +1782,13 @@ _PyXI_Exit(_PyXI_session *session)
 PyStatus
 _PyXI_Init(PyInterpreterState *interp)
 {
+    _PyXI_state_t *state = _PyXI_GET_STATE(interp);
+
     // Initialize the XID lookup state (e.g. registry).
     if (_Py_IsMainInterpreter(interp)) {
         xid_lookup_init(&_PyXI_GET_GLOBAL_STATE(interp)->data_lookup);
     }
-    xid_lookup_init(&_PyXI_GET_STATE(interp)->data_lookup);
+    xid_lookup_init(&state->data_lookup);
 
     // Initialize exceptions.(heap types).
     // See _PyXI_InitTypes() for the static types.
@@ -1801,12 +1806,14 @@ _PyXI_Init(PyInterpreterState *interp)
 void
 _PyXI_Fini(PyInterpreterState *interp)
 {
+    _PyXI_state_t *state = _PyXI_GET_STATE(interp);
+
     // Finalize exceptions (heap types).
     // See _PyXI_FiniTypes() for the static types.
     fini_heap_exctypes(&_PyXI_GET_STATE(interp)->exceptions);
 
     // Finalize the XID lookup state (e.g. registry).
-    xid_lookup_fini(&_PyXI_GET_STATE(interp)->data_lookup);
+    xid_lookup_fini(&state->data_lookup);
     if (_Py_IsMainInterpreter(interp)) {
         xid_lookup_fini(&_PyXI_GET_GLOBAL_STATE(interp)->data_lookup);
     }

Python/crossinterp_data_lookup.h

@@ -1,8 +1,8 @@
 #include "pycore_weakref.h"       // _PyWeakref_GET_REF()
 
 
-typedef struct _xidregistry dlregistry_t;
-typedef struct _xidregitem dlregitem_t;
+typedef _PyXIData_registry_t dlregistry_t;
+typedef _PyXIData_regitem_t dlregitem_t;
 
 
 // forward

Python/parking_lot.c

@@ -221,8 +221,7 @@ _PySemaphore_Wait(_PySemaphore *sema, PyTime_t timeout, int detach)
     PyThreadState *tstate = NULL;
     if (detach) {
         tstate = _PyThreadState_GET();
-        if (tstate && _Py_atomic_load_int_relaxed(&tstate->state) ==
-                          _Py_THREAD_ATTACHED) {
+        if (tstate && _PyThreadState_IsAttached(tstate)) {
             // Only detach if we are attached
             PyEval_ReleaseThread(tstate);
         }