gh-112532: Require mimalloc in --disable-gil builds

In `--disable-gil` builds, the default allocator is now "mimalloc" and the "malloc" and "pymalloc" allocators are disabled.
python · corona10 · Dec 12, 2023 · Dec 8, 2023 · Dec 8, 2023 · Dec 8, 2023
commit e7c5b9124f75e04f4635bdce2a306d693d7a69c6
@@ -18,7 +18,18 @@ extern void * _PyMem_RawRealloc(void *, void *, size_t);
 extern void _PyMem_RawFree(void *, void *);
 #define PYRAW_ALLOC {NULL, _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree}

-#if defined(WITH_PYMALLOC)
+#if defined(Py_GIL_DISABLED) && defined(WITH_MIMALLOC)
+extern void* _PyObject_MiMalloc(void *, size_t);
+extern void* _PyObject_MiCalloc(void *, size_t, size_t);
+extern void _PyObject_MiFree(void *, void *);
+extern void* _PyObject_MiRealloc(void *, void *, size_t);
+#  define PYOBJ_ALLOC {NULL, _PyObject_MiMalloc, _PyObject_MiCalloc, _PyObject_MiRealloc, _PyObject_MiFree}
+extern void* _PyMem_MiMalloc(void *, size_t);
+extern void* _PyMem_MiCalloc(void *, size_t, size_t);
+extern void _PyMem_MiFree(void *, void *);
+extern void* _PyMem_MiRealloc(void *, void *, size_t);
+#  define PYMEM_ALLOC {NULL, _PyMem_MiMalloc, _PyMem_MiCalloc, _PyMem_MiRealloc, _PyMem_MiFree}
+#elif defined(WITH_PYMALLOC)
 extern void* _PyObject_Malloc(void *, size_t);
 extern void* _PyObject_Calloc(void *, size_t, size_t);
 extern void _PyObject_Free(void *, void *);

diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
@@ -1844,7 +1844,7 @@ def restore(self):

 def with_pymalloc():
    import _testcapi
-    return _testcapi.WITH_PYMALLOC
+    return _testcapi.WITH_PYMALLOC and not Py_GIL_DISABLED


 def with_mimalloc():

diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py
@@ -152,6 +152,8 @@ class C(): pass
            self.assertGreaterEqual(count, i*5-2)


+# Py_GIL_DISABLED requires mimalloc (not malloc)
+@unittest.skipIf(support.Py_GIL_DISABLED, 'need malloc')
 class PyMemMallocDebugTests(PyMemDebugTests):
    PYTHONMALLOC = 'malloc_debug'

@@ -161,6 +163,11 @@ class PyMemPymallocDebugTests(PyMemDebugTests):
    PYTHONMALLOC = 'pymalloc_debug'


+@unittest.skipUnless(support.with_mimalloc(), 'need mimaloc')
+class PyMemMallocDebugTests(PyMemDebugTests):
+    PYTHONMALLOC = 'mimalloc_debug'
+
+
 @unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG')
 class PyMemDefaultTests(PyMemDebugTests):
    # test default allocator of Python compiled in debug mode

diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py
@@ -738,6 +738,8 @@ def test_xdev(self):
                out = self.run_xdev("-c", code, check_exitcode=False)
            if support.with_pymalloc():
                alloc_name = "pymalloc_debug"
+            elif support.Py_GIL_DISABLED:
+                alloc_name = "mimalloc_debug"
            else:
                alloc_name = "malloc_debug"
            self.assertEqual(out, alloc_name)
@@ -814,9 +816,13 @@ def check_pythonmalloc(self, env_var, name):
    @support.cpython_only
    def test_pythonmalloc(self):
        # Test the PYTHONMALLOC environment variable
+        malloc = not support.Py_GIL_DISABLED
        pymalloc = support.with_pymalloc()
        mimalloc = support.with_mimalloc()
-        if pymalloc:
+        if support.Py_GIL_DISABLED:
+            default_name = 'mimalloc_debug' if support.Py_DEBUG else 'mimalloc'
+            default_name_debug = 'mimalloc_debug'
+        elif pymalloc:
            default_name = 'pymalloc_debug' if support.Py_DEBUG else 'pymalloc'
            default_name_debug = 'pymalloc_debug'
        else:
@@ -826,9 +832,12 @@ def test_pythonmalloc(self):
        tests = [
            (None, default_name),
            ('debug', default_name_debug),
-            ('malloc', 'malloc'),
-            ('malloc_debug', 'malloc_debug'),
        ]
+        if malloc:
+            tests.extend([
+                ('malloc', 'malloc'),
+                ('malloc_debug', 'malloc_debug'),
+            ])
        if pymalloc:
            tests.extend((
                ('pymalloc', 'pymalloc'),

diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
@@ -23,6 +23,12 @@
 PYMEM_ALLOCATOR_NOT_SET = 0
 PYMEM_ALLOCATOR_DEBUG = 2
 PYMEM_ALLOCATOR_MALLOC = 3
+PYMEM_ALLOCATOR_MIMALLOC = 7
+if support.Py_GIL_DISABLED:
+    ALLOCATOR_FOR_CONFIG = PYMEM_ALLOCATOR_MIMALLOC
+else:
+    ALLOCATOR_FOR_CONFIG = PYMEM_ALLOCATOR_MALLOC
+
 Py_STATS = hasattr(sys, '_stats_on')

 # _PyCoreConfig_InitCompatConfig()
@@ -841,7 +847,7 @@ def test_init_global_config(self):

    def test_init_from_config(self):
        preconfig = {
-            'allocator': PYMEM_ALLOCATOR_MALLOC,
+            'allocator': ALLOCATOR_FOR_CONFIG,
            'utf8_mode': 1,
        }
        config = {
@@ -908,7 +914,7 @@ def test_init_from_config(self):

    def test_init_compat_env(self):
        preconfig = {
-            'allocator': PYMEM_ALLOCATOR_MALLOC,
+            'allocator': ALLOCATOR_FOR_CONFIG,
        }
        config = {
            'use_hash_seed': 1,
@@ -942,7 +948,7 @@ def test_init_compat_env(self):

    def test_init_python_env(self):
        preconfig = {
-            'allocator': PYMEM_ALLOCATOR_MALLOC,
+            'allocator': ALLOCATOR_FOR_CONFIG,
            'utf8_mode': 1,
        }
        config = {
@@ -984,7 +990,7 @@ def test_init_env_dev_mode(self):
                               api=API_COMPAT)

    def test_init_env_dev_mode_alloc(self):
-        preconfig = dict(allocator=PYMEM_ALLOCATOR_MALLOC)
+        preconfig = dict(allocator=ALLOCATOR_FOR_CONFIG)
        config = dict(dev_mode=1,
                      faulthandler=1,
                      warnoptions=['default'])

diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
@@ -5080,7 +5080,10 @@ def test_fork(self):
                support.wait_process(pid, exitcode=0)
        """
        assert_python_ok("-c", code)
-        assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug")
+        if support.Py_GIL_DISABLED:
+            assert_python_ok("-c", code, PYTHONMALLOC="mimalloc_debug")
+        else:
+            assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug")

    @unittest.skipUnless(sys.platform in ("linux", "darwin"),
                         "Only Linux and macOS detect this today.")

diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
@@ -16,6 +16,10 @@
 #  include "mimalloc/internal.h"  // for stats
 #endif

+#if defined(Py_GIL_DISABLED) && !defined(WITH_MIMALLOC)
+#  error "Py_GIL_DISABLED requires WITH_MIMALLOC"
+#endif
+
 #undef  uint
 #define uint pymem_uint

@@ -153,7 +157,12 @@ void* _PyObject_Realloc(void *ctx, void *ptr, size_t size);
 #  define PYMALLOC_ALLOC {NULL, _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free}
 #endif  // WITH_PYMALLOC

-#if defined(WITH_PYMALLOC)
+#if defined(Py_GIL_DISABLED)
+// Py_GIL_DISABLED requires using mimalloc for "mem" and "obj" domains.
+#  define PYRAW_ALLOC MALLOC_ALLOC
+#  define PYMEM_ALLOC MIMALLOC_ALLOC
+#  define PYOBJ_ALLOC MIMALLOC_OBJALLOC
+#elif defined(WITH_PYMALLOC)
 #  define PYRAW_ALLOC MALLOC_ALLOC
 #  define PYMEM_ALLOC PYMALLOC_ALLOC
 #  define PYOBJ_ALLOC PYMALLOC_ALLOC
@@ -350,7 +359,7 @@ _PyMem_GetAllocatorName(const char *name, PyMemAllocatorName *allocator)
    else if (strcmp(name, "debug") == 0) {
        *allocator = PYMEM_ALLOCATOR_DEBUG;
    }
-#ifdef WITH_PYMALLOC
+#if defined(WITH_PYMALLOC) && !defined(Py_GIL_DISABLED)
    else if (strcmp(name, "pymalloc") == 0) {
        *allocator = PYMEM_ALLOCATOR_PYMALLOC;
    }
@@ -366,12 +375,14 @@ _PyMem_GetAllocatorName(const char *name, PyMemAllocatorName *allocator)
        *allocator = PYMEM_ALLOCATOR_MIMALLOC_DEBUG;
    }
 #endif
+#ifndef Py_GIL_DISABLED
    else if (strcmp(name, "malloc") == 0) {
        *allocator = PYMEM_ALLOCATOR_MALLOC;
    }
    else if (strcmp(name, "malloc_debug") == 0) {
        *allocator = PYMEM_ALLOCATOR_MALLOC_DEBUG;
    }
+#endif
    else {
        /* unknown allocator */
        return -1;

diff --git a/Programs/_testembed.c b/Programs/_testembed.c
@@ -576,7 +576,11 @@ static int test_init_from_config(void)
    _PyPreConfig_InitCompatConfig(&preconfig);

    putenv("PYTHONMALLOC=malloc_debug");
+#ifndef Py_GIL_DISABLED
    preconfig.allocator = PYMEM_ALLOCATOR_MALLOC;
+#else
+    preconfig.allocator = PYMEM_ALLOCATOR_MIMALLOC;
+#endif

    putenv("PYTHONUTF8=0");
    Py_UTF8Mode = 0;
@@ -765,7 +769,11 @@ static int test_init_dont_parse_argv(void)
 static void set_most_env_vars(void)
 {
    putenv("PYTHONHASHSEED=42");
+#ifndef Py_GIL_DISABLED
    putenv("PYTHONMALLOC=malloc");
+#else
+    putenv("PYTHONMALLOC=mimalloc");
+#endif
    putenv("PYTHONTRACEMALLOC=2");
    putenv("PYTHONPROFILEIMPORTTIME=1");
    putenv("PYTHONNODEBUGRANGES=1");
@@ -851,7 +859,11 @@ static int test_init_env_dev_mode_alloc(void)
    /* Test initialization from environment variables */
    Py_IgnoreEnvironmentFlag = 0;
    set_all_env_vars_dev_mode();
+#ifndef Py_GIL_DISABLED
    putenv("PYTHONMALLOC=malloc");
+#else
+    putenv("PYTHONMALLOC=mimalloc");
+#endif
    _testembed_Py_InitializeFromConfig();
    dump_config();
    Py_Finalize();

@@ -4558,6 +4558,8 @@ if test "$with_mimalloc" != no; then
  with_mimalloc=yes
  AC_DEFINE([WITH_MIMALLOC], [1], [Define if you want to compile in mimalloc memory allocator.])
  AC_SUBST([MIMALLOC_HEADERS], ['$(MIMALLOC_HEADERS)'])
+elif test "$disable_gil" = "yes"; then
+  AC_MSG_ERROR([--disable-gil requires mimalloc memory allocator (--with-mimalloc).])
 fi

 AC_MSG_RESULT([$with_mimalloc])