Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Appearance settings

Commit ff58c3c

Browse filesBrowse files
[3.13] gh-123022: Fix crash with Py_Initialize in background thread (GH-123052) (#123114)
Check that the current default heap is initialized in `_mi_os_get_aligned_hint` and `mi_os_claim_huge_pages`. The mimalloc function `_mi_os_get_aligned_hint` assumes that there is an initialized default heap. This is true for our main thread, but not for background threads. The problematic code path is usually called during initialization (i.e., `Py_Initialize`), but it may also be called if the program allocates large amounts of memory in total. The crash only affected the free-threaded build. (cherry picked from commit d061ffe) Co-authored-by: Sam Gross <colesbury@gmail.com>
1 parent 6001dea commit ff58c3c
Copy full SHA for ff58c3c

File tree

4 files changed

+40
-5
lines changed
Filter options

4 files changed

+40
-5
lines changed

‎Lib/test/test_embed.py

Copy file name to clipboardExpand all lines: Lib/test/test_embed.py
+8-1Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Run the tests in Programs/_testembed.c (tests for the CPython embedding APIs)
22
from test import support
3-
from test.support import import_helper, os_helper, MS_WINDOWS
3+
from test.support import import_helper, os_helper, threading_helper, MS_WINDOWS
44
import unittest
55

66
from collections import namedtuple
@@ -1792,6 +1792,13 @@ def test_init_main_interpreter_settings(self):
17921792

17931793
self.assertEqual(out, expected)
17941794

1795+
@threading_helper.requires_working_threading()
1796+
def test_init_in_background_thread(self):
1797+
# gh-123022: Check that running Py_Initialize() in a background
1798+
# thread doesn't crash.
1799+
out, err = self.run_embedded_interpreter("test_init_in_background_thread")
1800+
self.assertEqual(err, "")
1801+
17951802

17961803
class SetConfigTests(unittest.TestCase):
17971804
def test_set_config(self):
+2Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fix crash in free-threaded build when calling :c:func:`Py_Initialize` from
2+
a non-main thread.

‎Objects/mimalloc/os.c

Copy file name to clipboardExpand all lines: Objects/mimalloc/os.c
+12-4Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -115,8 +115,12 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size)
115115
if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize
116116
uintptr_t init = MI_HINT_BASE;
117117
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode
118-
uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap());
119-
init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB
118+
mi_heap_t* heap = mi_prim_get_default_heap();
119+
// gh-123022: default heap may not be initialized in CPython in background threads
120+
if (mi_heap_is_initialized(heap)) {
121+
uintptr_t r = _mi_heap_random_next(heap);
122+
init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB
123+
}
120124
#endif
121125
uintptr_t expected = hint + size;
122126
mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init);
@@ -553,8 +557,12 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
553557
// Initialize the start address after the 32TiB area
554558
start = ((uintptr_t)32 << 40); // 32TiB virtual start address
555559
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode
556-
uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap());
557-
start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB
560+
mi_heap_t* heap = mi_prim_get_default_heap();
561+
// gh-123022: default heap may not be initialized in CPython in background threads
562+
if (mi_heap_is_initialized(heap)) {
563+
uintptr_t r = _mi_heap_random_next(heap);
564+
start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB
565+
}
558566
#endif
559567
}
560568
end = start + size;

‎Programs/_testembed.c

Copy file name to clipboardExpand all lines: Programs/_testembed.c
+18Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#include <Python.h>
99
#include "pycore_initconfig.h" // _PyConfig_InitCompatConfig()
1010
#include "pycore_runtime.h" // _PyRuntime
11+
#include "pycore_pythread.h" // PyThread_start_joinable_thread()
1112
#include "pycore_import.h" // _PyImport_FrozenBootstrap
1213
#include <inttypes.h>
1314
#include <stdio.h>
@@ -2022,6 +2023,22 @@ static int test_init_main_interpreter_settings(void)
20222023
return 0;
20232024
}
20242025

2026+
static void do_init(void *unused)
2027+
{
2028+
_testembed_Py_Initialize();
2029+
Py_Finalize();
2030+
}
2031+
2032+
static int test_init_in_background_thread(void)
2033+
{
2034+
PyThread_handle_t handle;
2035+
PyThread_ident_t ident;
2036+
if (PyThread_start_joinable_thread(&do_init, NULL, &ident, &handle) < 0) {
2037+
return -1;
2038+
}
2039+
return PyThread_join_thread(handle);
2040+
}
2041+
20252042

20262043
#ifndef MS_WINDOWS
20272044
#include "test_frozenmain.h" // M_test_frozenmain
@@ -2211,6 +2228,7 @@ static struct TestCase TestCases[] = {
22112228
{"test_get_argc_argv", test_get_argc_argv},
22122229
{"test_init_use_frozen_modules", test_init_use_frozen_modules},
22132230
{"test_init_main_interpreter_settings", test_init_main_interpreter_settings},
2231+
{"test_init_in_background_thread", test_init_in_background_thread},
22142232

22152233
// Audit
22162234
{"test_open_code_hook", test_open_code_hook},

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.