[WIP] bpo-1635741: Py_Finalize() finalizes builtin static types

Include/internal/pycore_object.h

@@ -15,6 +15,19 @@ extern "C" {
 PyAPI_FUNC(int) _PyType_CheckConsistency(PyTypeObject *type);
 PyAPI_FUNC(int) _PyDict_CheckConsistency(PyObject *mp, int check_content);
 
+// Finalize a static type: it must not be used after this call.
+// Once a static type is finalized, it must no longer be used.
+// Use assert(!_PyStaticType_IsFinalized(type)); to ensure that
+// a static type is not finalized.
+extern void _PyStaticType_Fini(PyTypeObject *type);
+
+#ifndef NDEBUG
+// Test if _PyStaticType_Fini() was called on a static type.
+// Always return 0 for heap types.
+// Usage: assert(!_PyStaticType_IsFinalized(type));
+extern int _PyStaticType_IsFinalized(PyTypeObject *type);
+#endif
+
 /* Tell the GC to track this object.
  *
  * NB: While the object is tracked by the collector, it must be safe to call the

Include/internal/pycore_pylifecycle.h

@@ -50,7 +50,8 @@ extern PyStatus _PyImportHooks_Init(PyThreadState *tstate);
 extern int _PyFloat_Init(void);
 extern PyStatus _Py_HashRandomization_Init(const PyConfig *);
 
-extern PyStatus _PyTypes_Init(void);
+extern PyStatus _PyStaticTypes_Init(void);
+extern void _PyStaticTypes_Fini(void);
 extern PyStatus _PyTypes_InitSlotDefs(void);
 extern PyStatus _PyImportZip_Init(PyThreadState *tstate);
 extern PyStatus _PyGC_Init(PyThreadState *tstate);

Objects/exceptions.c

@@ -2740,11 +2740,77 @@ _PyBuiltins_AddExceptions(PyObject *bltinmod)
 #undef INIT_ALIAS
 }
 
+// Finalize exceptions initialized by _PyExc_Init().
 void
 _PyExc_Fini(void)
 {
     free_preallocated_memerrors();
     Py_CLEAR(errnomap);
+
+    _PyStaticType_Fini(&_PyExc_BaseException);
+    _PyStaticType_Fini(&_PyExc_Exception);
+    _PyStaticType_Fini(&_PyExc_TypeError);
+    _PyStaticType_Fini(&_PyExc_StopAsyncIteration);
+    _PyStaticType_Fini(&_PyExc_StopIteration);
+    _PyStaticType_Fini(&_PyExc_GeneratorExit);
+    _PyStaticType_Fini(&_PyExc_SystemExit);
+    _PyStaticType_Fini(&_PyExc_KeyboardInterrupt);
+    _PyStaticType_Fini(&_PyExc_ImportError);
+    _PyStaticType_Fini(&_PyExc_ModuleNotFoundError);
+    _PyStaticType_Fini(&_PyExc_OSError);
+    _PyStaticType_Fini(&_PyExc_EOFError);
+    _PyStaticType_Fini(&_PyExc_RuntimeError);
+    _PyStaticType_Fini(&_PyExc_RecursionError);
+    _PyStaticType_Fini(&_PyExc_NotImplementedError);
+    _PyStaticType_Fini(&_PyExc_NameError);
+    _PyStaticType_Fini(&_PyExc_UnboundLocalError);
+    _PyStaticType_Fini(&_PyExc_AttributeError);
+    _PyStaticType_Fini(&_PyExc_SyntaxError);
+    _PyStaticType_Fini(&_PyExc_IndentationError);
+    _PyStaticType_Fini(&_PyExc_TabError);
+    _PyStaticType_Fini(&_PyExc_LookupError);
+    _PyStaticType_Fini(&_PyExc_IndexError);
+    _PyStaticType_Fini(&_PyExc_KeyError);
+    _PyStaticType_Fini(&_PyExc_ValueError);
+    _PyStaticType_Fini(&_PyExc_UnicodeError);
+    _PyStaticType_Fini(&_PyExc_UnicodeEncodeError);
+    _PyStaticType_Fini(&_PyExc_UnicodeDecodeError);
+    _PyStaticType_Fini(&_PyExc_UnicodeTranslateError);
+    _PyStaticType_Fini(&_PyExc_AssertionError);
+    _PyStaticType_Fini(&_PyExc_ArithmeticError);
+    _PyStaticType_Fini(&_PyExc_FloatingPointError);
+    _PyStaticType_Fini(&_PyExc_OverflowError);
+    _PyStaticType_Fini(&_PyExc_ZeroDivisionError);
+    _PyStaticType_Fini(&_PyExc_SystemError);
+    _PyStaticType_Fini(&_PyExc_ReferenceError);
+    _PyStaticType_Fini(&_PyExc_MemoryError);
+    _PyStaticType_Fini(&_PyExc_BufferError);
+    _PyStaticType_Fini(&_PyExc_Warning);
+    _PyStaticType_Fini(&_PyExc_UserWarning);
+    _PyStaticType_Fini(&_PyExc_DeprecationWarning);
+    _PyStaticType_Fini(&_PyExc_PendingDeprecationWarning);
+    _PyStaticType_Fini(&_PyExc_SyntaxWarning);
+    _PyStaticType_Fini(&_PyExc_RuntimeWarning);
+    _PyStaticType_Fini(&_PyExc_FutureWarning);
+    _PyStaticType_Fini(&_PyExc_ImportWarning);
+    _PyStaticType_Fini(&_PyExc_UnicodeWarning);
+    _PyStaticType_Fini(&_PyExc_BytesWarning);
+    _PyStaticType_Fini(&_PyExc_ResourceWarning);
+    _PyStaticType_Fini(&_PyExc_ConnectionError);
+    _PyStaticType_Fini(&_PyExc_BlockingIOError);
+    _PyStaticType_Fini(&_PyExc_BrokenPipeError);
+    _PyStaticType_Fini(&_PyExc_ChildProcessError);
+    _PyStaticType_Fini(&_PyExc_ConnectionAbortedError);
+    _PyStaticType_Fini(&_PyExc_ConnectionRefusedError);
+    _PyStaticType_Fini(&_PyExc_ConnectionResetError);
+    _PyStaticType_Fini(&_PyExc_FileExistsError);
+    _PyStaticType_Fini(&_PyExc_FileNotFoundError);
+    _PyStaticType_Fini(&_PyExc_IsADirectoryError);
+    _PyStaticType_Fini(&_PyExc_NotADirectoryError);
+    _PyStaticType_Fini(&_PyExc_InterruptedError);
+    _PyStaticType_Fini(&_PyExc_PermissionError);
+    _PyStaticType_Fini(&_PyExc_ProcessLookupError);
+    _PyStaticType_Fini(&_PyExc_TimeoutError);
 }
 
 /* Helper to do the equivalent of "raise X from Y" in C, but always using

Objects/object.c

@@ -1731,7 +1731,7 @@ PyObject _Py_NotImplementedStruct = {
 };
 
 PyStatus
-_PyTypes_Init(void)
+_PyStaticTypes_Init(void)
 {
     PyStatus status = _PyTypes_InitSlotDefs();
     if (_PyStatus_EXCEPTION(status)) {
@@ -1818,6 +1818,84 @@ _PyTypes_Init(void)
 }
 
 
+// Finalize static types initialized by _PyStaticTypes_Init().
+void
+_PyStaticTypes_Fini(void)
+{
+    _PyStaticType_Fini(&_PyWeakref_RefType);
+    _PyStaticType_Fini(&_PyWeakref_CallableProxyType);
+    _PyStaticType_Fini(&_PyWeakref_ProxyType);
+    _PyStaticType_Fini(&PyLong_Type);
+    _PyStaticType_Fini(&PyBool_Type);
+    _PyStaticType_Fini(&PyByteArray_Type);
+    _PyStaticType_Fini(&PyBytes_Type);
+    _PyStaticType_Fini(&PyList_Type);
+    _PyStaticType_Fini(&_PyNone_Type);
+    _PyStaticType_Fini(&_PyNotImplemented_Type);
+    _PyStaticType_Fini(&PyTraceBack_Type);
+    _PyStaticType_Fini(&PySuper_Type);
+    _PyStaticType_Fini(&PyRange_Type);
+    _PyStaticType_Fini(&PyDict_Type);
+    _PyStaticType_Fini(&PyDictKeys_Type);
+    _PyStaticType_Fini(&PyDictValues_Type);
+    _PyStaticType_Fini(&PyDictItems_Type);
+    _PyStaticType_Fini(&PyDictRevIterKey_Type);
+    _PyStaticType_Fini(&PyDictRevIterValue_Type);
+    _PyStaticType_Fini(&PyDictRevIterItem_Type);
+    _PyStaticType_Fini(&PyODict_Type);
+    _PyStaticType_Fini(&PyODictKeys_Type);
+    _PyStaticType_Fini(&PyODictItems_Type);
+    _PyStaticType_Fini(&PyODictValues_Type);
+    _PyStaticType_Fini(&PyODictIter_Type);
+    _PyStaticType_Fini(&PySet_Type);
+    _PyStaticType_Fini(&PyUnicode_Type);
+    _PyStaticType_Fini(&PySlice_Type);
+    _PyStaticType_Fini(&PyStaticMethod_Type);
+    _PyStaticType_Fini(&PyComplex_Type);
+    _PyStaticType_Fini(&PyFloat_Type);
+    _PyStaticType_Fini(&PyFrozenSet_Type);
+    _PyStaticType_Fini(&PyProperty_Type);
+    _PyStaticType_Fini(&_PyManagedBuffer_Type);
+    _PyStaticType_Fini(&PyMemoryView_Type);
+    _PyStaticType_Fini(&PyTuple_Type);
+    _PyStaticType_Fini(&PyEnum_Type);
+    _PyStaticType_Fini(&PyReversed_Type);
+    _PyStaticType_Fini(&PyStdPrinter_Type);
+    _PyStaticType_Fini(&PyCode_Type);
+    _PyStaticType_Fini(&PyFrame_Type);
+    _PyStaticType_Fini(&PyCFunction_Type);
+    _PyStaticType_Fini(&PyCMethod_Type);
+    _PyStaticType_Fini(&PyMethod_Type);
+    _PyStaticType_Fini(&PyFunction_Type);
+    _PyStaticType_Fini(&PyDictProxy_Type);
+    _PyStaticType_Fini(&PyGen_Type);
+    _PyStaticType_Fini(&PyGetSetDescr_Type);
+    _PyStaticType_Fini(&PyWrapperDescr_Type);
+    _PyStaticType_Fini(&_PyMethodWrapper_Type);
+    _PyStaticType_Fini(&PyEllipsis_Type);
+    _PyStaticType_Fini(&PyMemberDescr_Type);
+    _PyStaticType_Fini(&_PyNamespace_Type);
+    _PyStaticType_Fini(&PyCapsule_Type);
+    _PyStaticType_Fini(&PyLongRangeIter_Type);
+    _PyStaticType_Fini(&PyCell_Type);
+    _PyStaticType_Fini(&PyInstanceMethod_Type);
+    _PyStaticType_Fini(&PyClassMethodDescr_Type);
+    _PyStaticType_Fini(&PyMethodDescr_Type);
+    _PyStaticType_Fini(&PyCallIter_Type);
+    _PyStaticType_Fini(&PySeqIter_Type);
+    _PyStaticType_Fini(&PyPickleBuffer_Type);
+    _PyStaticType_Fini(&PyCoro_Type);
+    _PyStaticType_Fini(&_PyCoroWrapper_Type);
+    _PyStaticType_Fini(&_PyInterpreterID_Type);
+
+    // Finish by core types: object and type.
+    _PyStaticType_Fini(&PyType_Type);
+    _PyStaticType_Fini(&PyBaseObject_Type);
+
+    PyType_ClearCache();
+}
+
+
 void
 _Py_NewReference(PyObject *op)
 {

Objects/typeobject.c

@@ -173,6 +173,29 @@ _PyType_CheckConsistency(PyTypeObject *type)
 #undef CHECK
 }
 
+
+#ifndef NDEBUG
+int
+_PyStaticType_IsFinalized(PyTypeObject *type)
+{
+    if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) {
+        if (_PyRuntimeState_GetFinalizing(&_PyRuntime)) {
+            // _PyStaticType_Fini() sets tp_dict to NULL
+            return (type->tp_dict == NULL);
+        }
+        else {
+            return 0;
+        }
+    }
+    else {
+        // Assume that a heap type is not finalized if it's still accessible
+        return 0;
+    }
+
+}
+#endif
+
+
 static const char *
 _PyType_DocWithoutSignature(const char *name, const char *internal_doc)
 {
@@ -983,6 +1006,7 @@ type_call(PyTypeObject *type, PyObject *args, PyObject *kwds)
        caller loses its exception */
     assert(!_PyErr_Occurred(tstate));
 #endif
+    assert(!_PyStaticType_IsFinalized(type));
 
     /* Special case: type(x) should return Py_TYPE(x) */
     /* We only want type itself to accept the one-argument form (#27157) */
@@ -5362,6 +5386,8 @@ static int add_operators(PyTypeObject *);
 int
 PyType_Ready(PyTypeObject *type)
 {
+    assert(!_PyStaticType_IsFinalized(type));
+
     PyObject *dict, *bases;
     PyTypeObject *base;
     Py_ssize_t i, n;
@@ -5593,6 +5619,30 @@ PyType_Ready(PyTypeObject *type)
     return -1;
 }
 
+
+void
+_PyStaticType_Fini(PyTypeObject *type)
+{
+    assert(!(type->tp_flags & Py_TPFLAGS_HEAPTYPE));
+
+#ifdef Py_TRACE_REFS
+    _Py_ForgetReference((PyObject *)type);
+#endif
+
+    Py_CLEAR(type->tp_dict);
+    Py_CLEAR(type->tp_bases);
+    Py_CLEAR(type->tp_mro);
+    Py_CLEAR(type->tp_cache);
+    Py_CLEAR(type->tp_subclasses);
+
+    // Clear Py_TPFLAGS_READY flag
+    type->tp_flags &= ~Py_TPFLAGS_READY;
+
+    // Test that _PyStaticType_IsFinalized() works as expected
+    assert(_PyStaticType_IsFinalized(type));
+}
+
+
 static int
 add_subclass(PyTypeObject *base, PyTypeObject *type)
 {

Python/pylifecycle.c

@@ -586,7 +586,7 @@ pycore_init_types(PyThreadState *tstate)
     }
 
     if (is_main_interp) {
-        status = _PyTypes_Init();
+        status = _PyStaticTypes_Init();
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
@@ -1253,6 +1253,12 @@ flush_std_files(void)
 static void
 finalize_interp_types(PyThreadState *tstate, int is_main_interp)
 {
+    if (is_main_interp) {
+        _PyExc_Fini();
+        _PyStaticTypes_Fini();
+        // after this point, builtin static types must no longer be used
+    }
+
     _PyFrame_Fini(tstate);
     _PyAsyncGen_Fini(tstate);
     _PyContext_Fini(tstate);
@@ -1302,7 +1308,6 @@ finalize_interp_clear(PyThreadState *tstate)
 
     if (is_main_interp) {
         PyGrammar_RemoveAccelerators(&_PyParser_Grammar);
-        _PyExc_Fini();
     }
 
     finalize_interp_types(tstate, is_main_interp);