Restore compatibility with Python 3.11

* PyFrameObjects are now lazily allocated and no longer part of the
  PyThreadState. Therefore switch the _PyCFrame instead, but keep the
  PyFrameObject* for GC.

  python/cpython@ae0a2b7

* The recursion_depth is no longer stored directly in the PyThreadState
  and needs to be calculated from recursion_remaining and
  recursion_limit.

  python/cpython@b931077

* The exc_state was also simplified (no more exc_type and
  exc_traceback).

  python/cpython@396b583

* Finally the frame "data stack" needs to be saved and restored. How this
  was done in python-greenlet was used as a reference.

  python-greenlet/greenlet#280

* Add new test test_clean_callstack to check that the call stack from
  the creation of the Fiber doesn't leak into it (i.e.
  `tstate->cframe->current_frame = NULL;` in `stacklet__callback`;
  `tstate->frame = NULL;` in older Python versions).

Author: jhasse

.github/workflows/python-package.yml

@@ -20,7 +20,7 @@ jobs:
         # would use windows-latest, but distutils tries to build against the version of MSVC that python was compiled with
         #   https://github.com/pypa/setuptools/blob/main/setuptools/_distutils/msvc9compiler.py#L403
         os: [ "ubuntu-latest", "macos-12", "windows-2019" ]
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
         #os: [ "ubuntu-latest" ]
         #python-version: ["3.8"]
 

src/fibers.c

@@ -221,17 +221,32 @@ stacklet__callback(stacklet_handle h, void *arg)
     /* set current thread state before starting this new Fiber */
     tstate = PyThreadState_Get();
     ASSERT(tstate != NULL);
+    tstate->exc_state.exc_value = NULL;
+#if PY_MINOR_VERSION < 11
     tstate->frame = NULL;
     tstate->exc_state.exc_type = NULL;
-    tstate->exc_state.exc_value = NULL;
     tstate->exc_state.exc_traceback = NULL;
+#else
+    tstate->datastack_chunk = NULL;
+    tstate->datastack_top = NULL;
+    tstate->datastack_limit = NULL;
+    tstate->cframe->current_frame = NULL;
+#endif
     tstate->exc_state.previous_item = NULL;
 
-    self->ts.recursion_depth = tstate->recursion_depth;
     self->ts.frame = NULL;
-    self->ts.exc_state.exc_type = NULL;
     self->ts.exc_state.exc_value = NULL;
+#if PY_MINOR_VERSION < 11
+    self->ts.recursion_depth = tstate->recursion_depth;
+    self->ts.exc_state.exc_type = NULL;
     self->ts.exc_state.exc_traceback = NULL;
+#else
+    self->ts.recursion_depth = tstate->recursion_limit - tstate->recursion_remaining;
+    self->ts.cframe = NULL;
+    self->ts.datastack_chunk = NULL;
+    self->ts.datastack_top = NULL;
+    self->ts.datastack_limit = NULL;
+#endif
     self->ts.exc_state.previous_item = NULL;
 
     if (value == NULL) {
@@ -285,11 +300,21 @@ do_switch(Fiber *self, PyObject *value)
     tstate = PyThreadState_Get();
     ASSERT(tstate != NULL);
     ASSERT(tstate->dict != NULL);
+    current->ts.exc_state.exc_value = tstate->exc_state.exc_value;
+#if PY_MINOR_VERSION < 11
     current->ts.recursion_depth = tstate->recursion_depth;
     current->ts.frame = tstate->frame;
     current->ts.exc_state.exc_type = tstate->exc_state.exc_type;
-    current->ts.exc_state.exc_value = tstate->exc_state.exc_value;
     current->ts.exc_state.exc_traceback = tstate->exc_state.exc_traceback;
+#else
+    current->ts.recursion_depth = tstate->recursion_limit - tstate->recursion_remaining;
+    current->ts.frame = PyThreadState_GetFrame(tstate);
+    Py_XDECREF(current->ts.frame);
+    current->ts.cframe = tstate->cframe;
+    current->ts.datastack_chunk = tstate->datastack_chunk;
+    current->ts.datastack_top = tstate->datastack_top;
+    current->ts.datastack_limit = tstate->datastack_limit;
+#endif
     current->ts.exc_state.previous_item = tstate->exc_state.previous_item;
     ASSERT(current->stacklet_h == NULL);
 
@@ -328,17 +353,32 @@ do_switch(Fiber *self, PyObject *value)
     }
 
     /* restore state */
+    tstate->exc_state.exc_value = current->ts.exc_state.exc_value;
+#if PY_MINOR_VERSION < 11
     tstate->recursion_depth = current->ts.recursion_depth;
     tstate->frame = current->ts.frame;
     tstate->exc_state.exc_type = current->ts.exc_state.exc_type;
-    tstate->exc_state.exc_value = current->ts.exc_state.exc_value;
     tstate->exc_state.exc_traceback = current->ts.exc_state.exc_traceback;
+#else
+    tstate->recursion_remaining = tstate->recursion_limit - current->ts.recursion_depth;
+    tstate->cframe = current->ts.cframe;
+    tstate->datastack_chunk = current->ts.datastack_chunk;
+    tstate->datastack_top = current->ts.datastack_top;
+    tstate->datastack_limit = current->ts.datastack_limit;
+#endif
     tstate->exc_state.previous_item = current->ts.exc_state.previous_item;
 
     current->ts.frame = NULL;
-    current->ts.exc_state.exc_type = NULL;
     current->ts.exc_state.exc_value = NULL;
+#if PY_MINOR_VERSION < 11
+    current->ts.exc_state.exc_type = NULL;
     current->ts.exc_state.exc_traceback = NULL;
+#else
+    current->ts.cframe = NULL;
+    current->ts.datastack_chunk = NULL;
+    current->ts.datastack_top = NULL;
+    current->ts.datastack_limit = NULL;
+#endif
     current->ts.exc_state.previous_item = NULL;
 
     return result;
@@ -590,9 +630,11 @@ Fiber_tp_traverse(Fiber *self, visitproc visit, void *arg)
     Py_VISIT(self->ts_dict);
     Py_VISIT(self->parent);
     Py_VISIT(self->ts.frame);
-    Py_VISIT(self->ts.exc_state.exc_type);
     Py_VISIT(self->ts.exc_state.exc_value);
+#if PY_MINOR_VERSION < 11
+    Py_VISIT(self->ts.exc_state.exc_type);
     Py_VISIT(self->ts.exc_state.exc_traceback);
+#endif
     Py_VISIT(self->ts.exc_state.previous_item);
 
     return 0;
@@ -609,9 +651,11 @@ Fiber_tp_clear(Fiber *self)
     Py_CLEAR(self->ts_dict);
     Py_CLEAR(self->parent);
     Py_CLEAR(self->ts.frame);
-    Py_CLEAR(self->ts.exc_state.exc_type);
     Py_CLEAR(self->ts.exc_state.exc_value);
+#if PY_MINOR_VERSION < 11
+    Py_CLEAR(self->ts.exc_state.exc_type);
     Py_CLEAR(self->ts.exc_state.exc_traceback);
+#endif
     Py_CLEAR(self->ts.exc_state.previous_item);
 
     return 0;

src/fibers.h

@@ -30,6 +30,12 @@ typedef struct _fiber {
     PyObject *args;
     PyObject *kwargs;
     struct {
+#if PY_MINOR_VERSION >= 11
+        _PyCFrame *cframe;
+        _PyStackChunk *datastack_chunk;
+        PyObject **datastack_top;
+        PyObject **datastack_limit;
+#endif
         struct _frame *frame;
         int recursion_depth;
         _PyErr_StackItem exc_state;

tests/test_fibers.py

@@ -6,6 +6,7 @@
 
 import os
 import sys
+import traceback
 
 import fibers
 from fibers import Fiber, current
@@ -69,6 +70,19 @@ def f():
         lst.append(4)
         assert lst == list(range(5))
 
+    def test_clean_callstack(self):
+        lst = []
+        
+        def f():
+            for line in traceback.format_stack():
+                lst.append(line)
+        f()
+        expected = [lst[-1]]
+        lst = []
+        g = Fiber(f)
+        g.switch()
+        assert lst == expected
+
     def test_two_children(self):
         lst = []