Fix subinterpreter cleanup.

python · ericsnowcurrently · May 19, 2025 · Jan 8, 2025 · Jan 8, 2025 · Jan 8, 2025
commit 025cc1723113f1ad0c1d100bd8d9a8d4ed26e19c
@@ -2054,12 +2054,23 @@ _Py_Finalize(_PyRuntimeState *runtime)
    int malloc_stats = tstate->interp->config.malloc_stats;
 #endif

+    /* Clean up any lingering subinterpreters.
+
+       Two preconditions need to be met here:
+
+        - This has to happen before _PyRuntimeState_SetFinalizing is
+          called, or else threads might get prematurely blocked.
+        - The world must not be stopped, as finalizers can run.
+    */
+    finalize_subinterpreters();
+
    /* Ensure that remaining threads are detached */
    _PyEval_StopTheWorldAll(runtime);

    /* Remaining daemon threads will be trapped in PyThread_hang_thread
       when they attempt to take the GIL (ex: PyEval_RestoreThread()). */
    _PyInterpreterState_SetFinalizing(tstate->interp, tstate);
+
    _PyRuntimeState_SetFinalizing(runtime, tstate);
    runtime->initialized = 0;
    runtime->core_initialized = 0;
@@ -2121,9 +2132,6 @@ _Py_Finalize(_PyRuntimeState *runtime)
    _PyImport_FiniExternal(tstate->interp);
    finalize_modules(tstate);

-    /* Clean up any lingering subinterpreters. */
-    finalize_subinterpreters();
-
    /* Print debug stats if any */
    _PyEval_Fini();

@@ -2438,14 +2446,15 @@ Py_EndInterpreter(PyThreadState *tstate)
    _Py_FinishPendingCalls(tstate);

    _PyAtExit_Call(tstate->interp);
-
-    if (tstate != interp->threads.head || tstate->next != NULL) {
-        Py_FatalError("not the last thread");
-    }
+    _PyRuntimeState *runtime = interp->runtime;
+    _PyEval_StopTheWorldAll(runtime);
+    PyThreadState *list = _PyThreadState_RemoveExcept(tstate);

    /* Remaining daemon threads will automatically exit
       when they attempt to take the GIL (ex: PyEval_RestoreThread()). */
    _PyInterpreterState_SetFinalizing(interp, tstate);
+    _PyEval_StartTheWorldAll(runtime);
+    _PyThreadState_DeleteList(list);

    // XXX Call something like _PyImport_Disable() here?

@@ -2476,6 +2485,8 @@ finalize_subinterpreters(void)
    PyInterpreterState *main_interp = _PyInterpreterState_Main();
    assert(final_tstate->interp == main_interp);
    _PyRuntimeState *runtime = main_interp->runtime;
+    assert(!runtime->stoptheworld.world_stopped);
+    assert(_PyRuntimeState_GetFinalizing(runtime) == NULL);
    struct pyinterpreters *interpreters = &runtime->interpreters;

    /* Get the first interpreter in the list. */
@@ -2504,27 +2515,18 @@ finalize_subinterpreters(void)

    /* Clean up all remaining subinterpreters. */
    while (interp != NULL) {
-        assert(!_PyInterpreterState_IsRunningMain(interp));
-
-        /* Find the tstate to use for fini.  We assume the interpreter
-           will have at most one tstate at this point. */
-        PyThreadState *tstate = interp->threads.head;
-        if (tstate != NULL) {
-            /* Ideally we would be able to use tstate as-is, and rely
-               on it being in a ready state: no exception set, not
-               running anything (tstate->current_frame), matching the
-               current thread ID (tstate->thread_id).  To play it safe,
-               we always delete it and use a fresh tstate instead. */
-            assert(tstate != final_tstate);
-            _PyThreadState_Attach(tstate);
-            PyThreadState_Clear(tstate);
-            _PyThreadState_Detach(tstate);
-            PyThreadState_Delete(tstate);
+        /* Make a tstate for finalization. */
+        PyThreadState *tstate = _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_FINI);
+        if (tstate == NULL)
+        {
+            // XXX Some graceful way to always get a thread state?
+            Py_FatalError("thread state allocation failed");
        }
-        tstate = _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_FINI);

-        /* Destroy the subinterpreter. */
+        /* Enter the subinterpreter. */
        _PyThreadState_Attach(tstate);
+
+        /* Destroy the subinterpreter. */
        Py_EndInterpreter(tstate);
        assert(_PyThreadState_GET() == NULL);