Phase 2

Adds the ability to make everything immutable and the `behaviors`
module, which contains multi-interpreter lock implementations.
Furthermore, module loading, unloading, and subclassing have been
modified to be BOC safe.

Signed-off-by: Matthew A Johnson <matjoh@microsoft.com>

Author: matajoh

Doc/c-api/type.rst

@@ -466,7 +466,7 @@ The following functions and structs are used to create
 
       * :c:member:`~PyTypeObject.tp_dict`
       * :c:member:`~PyTypeObject.tp_mro`
-      * :c:member:`~PyTypeObject.tp_cache`
+      * :c:member:`~PyTypeObject.tp_lock`
       * :c:member:`~PyTypeObject.tp_subclasses`
       * :c:member:`~PyTypeObject.tp_weaklist`
       * :c:member:`~PyTypeObject.tp_vectorcall`

Doc/c-api/typeobj.rst

@@ -133,7 +133,7 @@ Quick Reference
    +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+
    | <:c:member:`~PyTypeObject.tp_mro`>             | :c:type:`PyObject` *              | __mro__           |   |   | ~ |   |
    +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+
-   | [:c:member:`~PyTypeObject.tp_cache`]           | :c:type:`PyObject` *              |                   |   |   |       |
+   | [:c:member:`~PyTypeObject.tp_lock`]            | :c:type:`PyObject` *              |                   |   |   |       |
    +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+
    | [:c:member:`~PyTypeObject.tp_subclasses`]      | void *                            | __subclasses__    |   |   |       |
    +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+
@@ -2013,9 +2013,9 @@ and :c:data:`PyType_Type` effectively act as defaults.)
    :c:func:`PyType_Ready`.
 
 
-.. c:member:: PyObject* PyTypeObject.tp_cache
+.. c:member:: PyObject* PyTypeObject.tp_lock
 
-   Unused.  Internal use only.
+   Internal use only.
 
    **Inheritance:**
 

Doc/data/python3.12.abi

@@ -19363,7 +19363,7 @@
         <var-decl name='tp_mro' type-id='type-id-2' visibility='default' filepath='./Include/cpython/object.h' line='217' column='1'/>
       </data-member>
       <data-member access='public' layout-offset-in-bits='2816'>
-        <var-decl name='tp_cache' type-id='type-id-2' visibility='default' filepath='./Include/cpython/object.h' line='218' column='1'/>
+        <var-decl name='tp_lock' type-id='type-id-2' visibility='default' filepath='./Include/cpython/object.h' line='218' column='1'/>
       </data-member>
       <data-member access='public' layout-offset-in-bits='2880'>
         <var-decl name='tp_subclasses' type-id='type-id-22' visibility='default' filepath='./Include/cpython/object.h' line='219' column='1'/>

Doc/includes/typestruct.h

@@ -70,7 +70,7 @@ typedef struct _typeobject {
     inquiry tp_is_gc; /* For PyObject_IS_GC */
     PyObject *tp_bases;
     PyObject *tp_mro; /* method resolution order */
-    PyObject *tp_cache;
+    PyObject *tp_lock;
     PyObject *tp_subclasses;
     PyObject *tp_weaklist;
     destructor tp_del;

Include/cpython/object.h

@@ -215,7 +215,7 @@ struct _typeobject {
     inquiry tp_is_gc; /* For PyObject_IS_GC */
     PyObject *tp_bases;
     PyObject *tp_mro; /* method resolution order */
-    PyObject *tp_cache; /* no longer used */
+    PyObject *tp_lock; /* used by immutable types */
     void *tp_subclasses;  /* for static builtin types this is an index */
     PyObject *tp_weaklist; /* not used for static builtin types */
     destructor tp_del;

Include/internal/pycore_dict.h

@@ -40,7 +40,7 @@ typedef struct {
 #define _PyDictEntry_SetValue(entry, value) ((entry)->_me_value = value)
 #define _PyDictEntry_IsEmpty(entry) ((entry)->_me_value == NULL)
 
-extern bool _PyDict_IsKeyImmutable(PyObject* op, PyObject* key);
+extern PyObject *_PyDict_IsKeyImmutable(PyObject* op, PyObject* key);
 extern PyDictKeysObject *_PyDict_NewKeysForClass(void);
 extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *);
 

Include/internal/pycore_regions.h

@@ -17,6 +17,10 @@ extern "C" {
 PyObject* _Py_MakeImmutable(PyObject* obj);
 #define Py_MakeImmutable(op) _Py_MakeImmutable(_PyObject_CAST(op))
 
+PyObject* Py_MakeGlobalsImmutable(void);
+
+bool _PyGlobalsImmutable_Check(void);
+
 #ifdef NDEBUG
 #define _Py_VPYDBG(fmt, ...)
 #define _Py_VPYDBGPRINT(fmt, ...)

Lib/importlib/_bootstrap.py

@@ -29,6 +29,7 @@ def _object_name(obj):
 # Bootstrap-related code ######################################################
 
 # Modules injected manually by _setup()
+_behaviors = None
 _thread = None
 _warnings = None
 _weakref = None
@@ -407,6 +408,69 @@ def __repr__(self):
         return f'_DummyModuleLock({self.name!r}) at {id(self)}'
 
 
+class _MultiInterpreterModuleLock:
+    """A recursive lock implementation which is able to detect deadlocks
+    across multiple subinterpreters.
+    """
+
+    def __init__(self, name):
+        self.lock = _behaviors.RLock()
+        self.wakeup = _behaviors.Lock()
+        self.name = name
+        self.owner = None
+        self.count = []
+        self.waiters = []
+
+    def has_deadlock(self):
+        return _has_deadlocked(
+            # get the thread id of the current subinterpreter
+            target_id=_behaviors.get_ident(),
+            seen_ids=set(),
+            candidate_ids=[self.owner],
+            blocking_on=_blocking_on,
+        )
+
+    def acquire(self):
+        """
+        Acquire the module lock.  If a potential deadlock is detected,
+        a _DeadlockError is raised.
+        Otherwise, the lock is always acquired and True is returned.
+        """
+        bid = _behaviors.get_ident()
+        with _BlockingOnManager(bid, self):
+            while True:
+                with self.lock:
+                    if self.count == [] or self.owner == bid:
+                        self.owner = bid
+                        self.count.append(True)
+                        return True
+
+                    if self.has_deadlock():
+                        raise _DeadlockError(f'deadlock detected by {self!r}')
+
+                    if self.wakeup.acquire(False):
+                        self.waiters.append(None)
+
+                self.wakeup.acquire()
+                self.wakeup.release()
+
+    def release(self):
+        bid = _behaviors.get_ident()
+        with self.lock:
+            if self.owner != bid:
+                raise RuntimeError('cannot release un-acquired lock')
+            assert len(self.count) > 0
+            self.count.pop()
+            if not len(self.count):
+                self.owner = None
+                if len(self.waiters) > 0:
+                    self.waiters.pop()
+                    self.wakeup.release()
+
+    def __repr__(self):
+        return f'_MultiInterpreterModuleLock({self.name!r}) at {id(self)}'
+
+
 class _ModuleLockManager:
 
     def __init__(self, name):
@@ -439,6 +503,8 @@ def _get_module_lock(name):
         if lock is None:
             if _thread is None:
                 lock = _DummyModuleLock(name)
+            elif _behaviors is not None and _behaviors.running():
+                lock = _MultiInterpreterModuleLock(name)
             else:
                 lock = _ModuleLock(name)
 
@@ -943,6 +1009,11 @@ def _load_unlocked(spec):
     finally:
         spec._initializing = False
 
+    if _behaviors is not None and _behaviors.running():
+        # all modules must be made immutable upon load
+        if not isimmutable(module):
+            makeimmutable(module)
+
     return module
 
 # A method used during testing of _load_unlocked() and by
@@ -1518,7 +1589,7 @@ def _setup(sys_module, _imp_module):
 
     # Directly load built-in modules needed during bootstrap.
     self_module = sys.modules[__name__]
-    for builtin_name in ('_thread', '_warnings', '_weakref'):
+    for builtin_name in ('_thread', '_warnings', '_weakref', '_behaviors'):
         if builtin_name not in sys.modules:
             builtin_module = _builtin_from_name(builtin_name)
         else:

Makefile.pre.in

@@ -431,6 +431,7 @@ PYTHON_OBJS=	\
 		Python/fileutils.o \
 		Python/suggestions.o \
 		Python/perf_trampoline.o \
+		Python/regions.o \
 		Python/$(DYNLOADFILE) \
 		$(LIBOBJS) \
 		$(MACHDEP_OBJS) \
@@ -475,7 +476,6 @@ OBJECT_OBJS=	\
 		Objects/obmalloc.o \
 		Objects/picklebufobject.o \
 		Objects/rangeobject.o \
-		Objects/regions.o \
 		Objects/setobject.o \
 		Objects/sliceobject.o \
 		Objects/structseq.o \

Modules/Setup

@@ -131,6 +131,7 @@ PYTHONPATH=$(COREPYTHONPATH)
 # Modules that should always be present (POSIX and Windows):
 
 #_asyncio _asynciomodule.c
+#_behaviors _behaviorsmodule.c
 #_bisect _bisectmodule.c
 #_contextvars _contextvarsmodule.c
 #_csv _csv.c