[WIP] bpo-1635741: Py_Finalize() finalizes builtin static types

Clear the following PyTypeObject members of builtin static types in Py_Finalize(): * tp_bases * tp_cache * tp_dict * tp_mro * tp_subclasses Finalize static types initialized by _PyStaticTypes_Init() and builtin exceptions initialized by _PyExc_Init(). Once a static type is finalized, it must no longer be used. Use assert(!_PyStaticType_IsFinalized(type)); to ensure that static type is not finalized. With this change, Valgrind log: possibly lost: 612,326 bytes in 5,503 blocks becomes: possibly lost: 369,346 bytes in 2,805 blocks" Changes: * Rename _PyTypes_Init() to _PyStaticTypes_Init(). * Move _PyExc_Fini() call inside finalize_interp_types().
python · vstinner · Jun 8, 2020 · Jun 9, 2020 · 530ccbf3eaae9ff67d53db2b2419b241ea5d673c
commit 530ccbf3eaae9ff67d53db2b2419b241ea5d673c
diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h
@@ -15,6 +15,19 @@ extern "C" {
 PyAPI_FUNC(int) _PyType_CheckConsistency(PyTypeObject *type);
 PyAPI_FUNC(int) _PyDict_CheckConsistency(PyObject *mp, int check_content);

+// Finalize a static type: it must not be used after this call.
+// Once a static type is finalized, it must no longer be used.
+// Use assert(!_PyStaticType_IsFinalized(type)); to ensure that
+// a static type is not finalized.
+extern void _PyStaticType_Fini(PyTypeObject *type);
+
+#ifndef NDEBUG
+// Test if _PyStaticType_Fini() was called on a static type.
+// Always return 0 for heap types.
+// Usage: assert(!_PyStaticType_IsFinalized(type));
+extern int _PyStaticType_IsFinalized(PyTypeObject *type);
+#endif
+
 /* Tell the GC to track this object.
 *
 * NB: While the object is tracked by the collector, it must be safe to call the

diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h
@@ -50,7 +50,8 @@ extern PyStatus _PyImportHooks_Init(PyThreadState *tstate);
 extern int _PyFloat_Init(void);
 extern PyStatus _Py_HashRandomization_Init(const PyConfig *);

-extern PyStatus _PyTypes_Init(void);
+extern PyStatus _PyStaticTypes_Init(void);
+extern void _PyStaticTypes_Fini(void);
 extern PyStatus _PyTypes_InitSlotDefs(void);
 extern PyStatus _PyImportZip_Init(PyThreadState *tstate);
 extern PyStatus _PyGC_Init(PyThreadState *tstate);

diff --git a/Objects/exceptions.c b/Objects/exceptions.c
@@ -2740,11 +2740,77 @@ _PyBuiltins_AddExceptions(PyObject *bltinmod)
 #undef INIT_ALIAS
 }

+// Finalize exceptions initialized by _PyExc_Init().
 void
 _PyExc_Fini(void)
 {
    free_preallocated_memerrors();
    Py_CLEAR(errnomap);
+
+    _PyStaticType_Fini(&_PyExc_BaseException);
+    _PyStaticType_Fini(&_PyExc_Exception);
+    _PyStaticType_Fini(&_PyExc_TypeError);
+    _PyStaticType_Fini(&_PyExc_StopAsyncIteration);
+    _PyStaticType_Fini(&_PyExc_StopIteration);
+    _PyStaticType_Fini(&_PyExc_GeneratorExit);
+    _PyStaticType_Fini(&_PyExc_SystemExit);
+    _PyStaticType_Fini(&_PyExc_KeyboardInterrupt);
+    _PyStaticType_Fini(&_PyExc_ImportError);
+    _PyStaticType_Fini(&_PyExc_ModuleNotFoundError);
+    _PyStaticType_Fini(&_PyExc_OSError);
+    _PyStaticType_Fini(&_PyExc_EOFError);
+    _PyStaticType_Fini(&_PyExc_RuntimeError);
+    _PyStaticType_Fini(&_PyExc_RecursionError);
+    _PyStaticType_Fini(&_PyExc_NotImplementedError);
+    _PyStaticType_Fini(&_PyExc_NameError);
+    _PyStaticType_Fini(&_PyExc_UnboundLocalError);
+    _PyStaticType_Fini(&_PyExc_AttributeError);
+    _PyStaticType_Fini(&_PyExc_SyntaxError);
+    _PyStaticType_Fini(&_PyExc_IndentationError);
+    _PyStaticType_Fini(&_PyExc_TabError);
+    _PyStaticType_Fini(&_PyExc_LookupError);
+    _PyStaticType_Fini(&_PyExc_IndexError);
+    _PyStaticType_Fini(&_PyExc_KeyError);
+    _PyStaticType_Fini(&_PyExc_ValueError);
+    _PyStaticType_Fini(&_PyExc_UnicodeError);
+    _PyStaticType_Fini(&_PyExc_UnicodeEncodeError);
+    _PyStaticType_Fini(&_PyExc_UnicodeDecodeError);
+    _PyStaticType_Fini(&_PyExc_UnicodeTranslateError);
+    _PyStaticType_Fini(&_PyExc_AssertionError);
+    _PyStaticType_Fini(&_PyExc_ArithmeticError);
+    _PyStaticType_Fini(&_PyExc_FloatingPointError);
+    _PyStaticType_Fini(&_PyExc_OverflowError);
+    _PyStaticType_Fini(&_PyExc_ZeroDivisionError);
+    _PyStaticType_Fini(&_PyExc_SystemError);
+    _PyStaticType_Fini(&_PyExc_ReferenceError);
+    _PyStaticType_Fini(&_PyExc_MemoryError);
+    _PyStaticType_Fini(&_PyExc_BufferError);
+    _PyStaticType_Fini(&_PyExc_Warning);
+    _PyStaticType_Fini(&_PyExc_UserWarning);
+    _PyStaticType_Fini(&_PyExc_DeprecationWarning);
+    _PyStaticType_Fini(&_PyExc_PendingDeprecationWarning);
+    _PyStaticType_Fini(&_PyExc_SyntaxWarning);
+    _PyStaticType_Fini(&_PyExc_RuntimeWarning);
+    _PyStaticType_Fini(&_PyExc_FutureWarning);
+    _PyStaticType_Fini(&_PyExc_ImportWarning);
+    _PyStaticType_Fini(&_PyExc_UnicodeWarning);
+    _PyStaticType_Fini(&_PyExc_BytesWarning);
+    _PyStaticType_Fini(&_PyExc_ResourceWarning);
+    _PyStaticType_Fini(&_PyExc_ConnectionError);
+    _PyStaticType_Fini(&_PyExc_BlockingIOError);
+    _PyStaticType_Fini(&_PyExc_BrokenPipeError);
+    _PyStaticType_Fini(&_PyExc_ChildProcessError);
+    _PyStaticType_Fini(&_PyExc_ConnectionAbortedError);
+    _PyStaticType_Fini(&_PyExc_ConnectionRefusedError);
+    _PyStaticType_Fini(&_PyExc_ConnectionResetError);
+    _PyStaticType_Fini(&_PyExc_FileExistsError);
+    _PyStaticType_Fini(&_PyExc_FileNotFoundError);
+    _PyStaticType_Fini(&_PyExc_IsADirectoryError);
+    _PyStaticType_Fini(&_PyExc_NotADirectoryError);
+    _PyStaticType_Fini(&_PyExc_InterruptedError);
+    _PyStaticType_Fini(&_PyExc_PermissionError);
+    _PyStaticType_Fini(&_PyExc_ProcessLookupError);
+    _PyStaticType_Fini(&_PyExc_TimeoutError);
 }

 /* Helper to do the equivalent of "raise X from Y" in C, but always using

diff --git a/Objects/object.c b/Objects/object.c
@@ -1731,7 +1731,7 @@ PyObject _Py_NotImplementedStruct = {
 };

 PyStatus
-_PyTypes_Init(void)
+_PyStaticTypes_Init(void)
 {
    PyStatus status = _PyTypes_InitSlotDefs();
    if (_PyStatus_EXCEPTION(status)) {
@@ -1818,6 +1818,84 @@ _PyTypes_Init(void)
 }


+// Finalize static types initialized by _PyStaticTypes_Init().
+void
+_PyStaticTypes_Fini(void)
+{
+    _PyStaticType_Fini(&_PyWeakref_RefType);
+    _PyStaticType_Fini(&_PyWeakref_CallableProxyType);
+    _PyStaticType_Fini(&_PyWeakref_ProxyType);
+    _PyStaticType_Fini(&PyLong_Type);
+    _PyStaticType_Fini(&PyBool_Type);
+    _PyStaticType_Fini(&PyByteArray_Type);
+    _PyStaticType_Fini(&PyBytes_Type);
+    _PyStaticType_Fini(&PyList_Type);
+    _PyStaticType_Fini(&_PyNone_Type);
+    _PyStaticType_Fini(&_PyNotImplemented_Type);
+    _PyStaticType_Fini(&PyTraceBack_Type);
+    _PyStaticType_Fini(&PySuper_Type);
+    _PyStaticType_Fini(&PyRange_Type);
+    _PyStaticType_Fini(&PyDict_Type);
+    _PyStaticType_Fini(&PyDictKeys_Type);
+    _PyStaticType_Fini(&PyDictValues_Type);
+    _PyStaticType_Fini(&PyDictItems_Type);
+    _PyStaticType_Fini(&PyDictRevIterKey_Type);
+    _PyStaticType_Fini(&PyDictRevIterValue_Type);
+    _PyStaticType_Fini(&PyDictRevIterItem_Type);
+    _PyStaticType_Fini(&PyODict_Type);
+    _PyStaticType_Fini(&PyODictKeys_Type);
+    _PyStaticType_Fini(&PyODictItems_Type);
+    _PyStaticType_Fini(&PyODictValues_Type);
+    _PyStaticType_Fini(&PyODictIter_Type);
+    _PyStaticType_Fini(&PySet_Type);
+    _PyStaticType_Fini(&PyUnicode_Type);
+    _PyStaticType_Fini(&PySlice_Type);
+    _PyStaticType_Fini(&PyStaticMethod_Type);
+    _PyStaticType_Fini(&PyComplex_Type);
+    _PyStaticType_Fini(&PyFloat_Type);
+    _PyStaticType_Fini(&PyFrozenSet_Type);
+    _PyStaticType_Fini(&PyProperty_Type);
+    _PyStaticType_Fini(&_PyManagedBuffer_Type);
+    _PyStaticType_Fini(&PyMemoryView_Type);
+    _PyStaticType_Fini(&PyTuple_Type);
+    _PyStaticType_Fini(&PyEnum_Type);
+    _PyStaticType_Fini(&PyReversed_Type);
+    _PyStaticType_Fini(&PyStdPrinter_Type);
+    _PyStaticType_Fini(&PyCode_Type);
+    _PyStaticType_Fini(&PyFrame_Type);
+    _PyStaticType_Fini(&PyCFunction_Type);
+    _PyStaticType_Fini(&PyCMethod_Type);
+    _PyStaticType_Fini(&PyMethod_Type);
+    _PyStaticType_Fini(&PyFunction_Type);
+    _PyStaticType_Fini(&PyDictProxy_Type);
+    _PyStaticType_Fini(&PyGen_Type);
+    _PyStaticType_Fini(&PyGetSetDescr_Type);
+    _PyStaticType_Fini(&PyWrapperDescr_Type);
+    _PyStaticType_Fini(&_PyMethodWrapper_Type);
+    _PyStaticType_Fini(&PyEllipsis_Type);
+    _PyStaticType_Fini(&PyMemberDescr_Type);
+    _PyStaticType_Fini(&_PyNamespace_Type);
+    _PyStaticType_Fini(&PyCapsule_Type);
+    _PyStaticType_Fini(&PyLongRangeIter_Type);
+    _PyStaticType_Fini(&PyCell_Type);
+    _PyStaticType_Fini(&PyInstanceMethod_Type);
+    _PyStaticType_Fini(&PyClassMethodDescr_Type);
+    _PyStaticType_Fini(&PyMethodDescr_Type);
+    _PyStaticType_Fini(&PyCallIter_Type);
+    _PyStaticType_Fini(&PySeqIter_Type);
+    _PyStaticType_Fini(&PyPickleBuffer_Type);
+    _PyStaticType_Fini(&PyCoro_Type);
+    _PyStaticType_Fini(&_PyCoroWrapper_Type);
+    _PyStaticType_Fini(&_PyInterpreterID_Type);
+
+    // Finish by core types: object and type.
+    _PyStaticType_Fini(&PyType_Type);
+    _PyStaticType_Fini(&PyBaseObject_Type);
+
+    PyType_ClearCache();
+}
+
+
 void
 _Py_NewReference(PyObject *op)
 {

diff --git a/Objects/typeobject.c b/Objects/typeobject.c
@@ -173,6 +173,29 @@ _PyType_CheckConsistency(PyTypeObject *type)
 #undef CHECK
 }

+
+#ifndef NDEBUG
+int
+_PyStaticType_IsFinalized(PyTypeObject *type)
+{
+    if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) {
+        if (_PyRuntimeState_GetFinalizing(&_PyRuntime)) {
+            // _PyStaticType_Fini() sets tp_dict to NULL
+            return (type->tp_dict == NULL);
+        }
+        else {
+            return 0;
+        }
+    }
+    else {
+        // Assume that a heap type is not finalized if it's still accessible
+        return 0;
+    }
+
+}
+#endif
+
+
 static const char *
 _PyType_DocWithoutSignature(const char *name, const char *internal_doc)
 {
@@ -983,6 +1006,7 @@ type_call(PyTypeObject *type, PyObject *args, PyObject *kwds)
       caller loses its exception */
    assert(!_PyErr_Occurred(tstate));
 #endif
+    assert(!_PyStaticType_IsFinalized(type));

    /* Special case: type(x) should return Py_TYPE(x) */
    /* We only want type itself to accept the one-argument form (#27157) */
@@ -5362,6 +5386,8 @@ static int add_operators(PyTypeObject *);
 int
 PyType_Ready(PyTypeObject *type)
 {
+    assert(!_PyStaticType_IsFinalized(type));
+
    PyObject *dict, *bases;
    PyTypeObject *base;
    Py_ssize_t i, n;
@@ -5593,6 +5619,30 @@ PyType_Ready(PyTypeObject *type)
    return -1;
 }

+
+void
+_PyStaticType_Fini(PyTypeObject *type)
+{
+    assert(!(type->tp_flags & Py_TPFLAGS_HEAPTYPE));
+
+#ifdef Py_TRACE_REFS
+    _Py_ForgetReference((PyObject *)type);
+#endif
+
+    Py_CLEAR(type->tp_dict);
+    Py_CLEAR(type->tp_bases);
+    Py_CLEAR(type->tp_mro);
+    Py_CLEAR(type->tp_cache);
+    Py_CLEAR(type->tp_subclasses);
+
+    // Clear Py_TPFLAGS_READY flag
+    type->tp_flags &= ~Py_TPFLAGS_READY;
+
+    // Test that _PyStaticType_IsFinalized() works as expected
+    assert(_PyStaticType_IsFinalized(type));
+}
+
+
 static int
 add_subclass(PyTypeObject *base, PyTypeObject *type)
 {

diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
@@ -586,7 +586,7 @@ pycore_init_types(PyThreadState *tstate)
    }

    if (is_main_interp) {
-        status = _PyTypes_Init();
+        status = _PyStaticTypes_Init();
        if (_PyStatus_EXCEPTION(status)) {
            return status;
        }
@@ -1253,6 +1253,12 @@ flush_std_files(void)
 static void
 finalize_interp_types(PyThreadState *tstate, int is_main_interp)
 {
+    if (is_main_interp) {
+        _PyExc_Fini();
+        _PyStaticTypes_Fini();
+        // after this point, builtin static types must no longer be used
+    }
+
    _PyFrame_Fini(tstate);
    _PyAsyncGen_Fini(tstate);
    _PyContext_Fini(tstate);
@@ -1302,7 +1308,6 @@ finalize_interp_clear(PyThreadState *tstate)

    if (is_main_interp) {
        PyGrammar_RemoveAccelerators(&_PyParser_Grammar);
-        _PyExc_Fini();
    }

    finalize_interp_types(tstate, is_main_interp);