Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit f77adf4

Browse filesBrowse files
daanxdaanx
committed
merge from dev (visit abandoned, upstream of python/cpython#114133)
2 parents 556b81b + 6b15342 commit f77adf4
Copy full SHA for f77adf4

21 files changed

+755
-315
lines changed

‎include/mimalloc.h

Copy file name to clipboardExpand all lines: include/mimalloc.h
+20-2Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -263,7 +263,7 @@ typedef struct mi_heap_area_s {
263263

264264
typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg);
265265

266-
mi_decl_export bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_all_blocks, mi_block_visit_fun* visitor, void* arg);
266+
mi_decl_export bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
267267

268268
// Experimental
269269
mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept;
@@ -289,8 +289,25 @@ mi_decl_export bool mi_manage_os_memory_ex(void* start, size_t size, bool is_co
289289
mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id);
290290
#endif
291291

292+
293+
// Experimental: allow sub-processes whose memory segments stay separated (and no reclamation between them)
294+
// Used for example for separate interpreter's in one process.
295+
typedef void* mi_subproc_id_t;
296+
mi_decl_export mi_subproc_id_t mi_subproc_main(void);
297+
mi_decl_export mi_subproc_id_t mi_subproc_new(void);
298+
mi_decl_export void mi_subproc_delete(mi_subproc_id_t subproc);
299+
mi_decl_export void mi_subproc_add_current_thread(mi_subproc_id_t subproc); // this should be called right after a thread is created (and no allocation has taken place yet)
300+
301+
// Experimental: visit abandoned heap areas (from threads that have been terminated)
302+
mi_decl_export bool mi_abandoned_visit_blocks(mi_subproc_id_t subproc_id, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
303+
304+
// Experimental: create a new heap with a specified heap tag. Set `allow_destroy` to false to allow the thread
305+
// to reclaim abandoned memory (with a compatible heap_tag and arena_id) but in that case `mi_heap_destroy` will
306+
// fall back to `mi_heap_delete`.
307+
mi_decl_export mi_decl_nodiscard mi_heap_t* mi_heap_new_ex(int heap_tag, bool allow_destroy, mi_arena_id_t arena_id);
308+
292309
// deprecated
293-
mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept;
310+
mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept;
294311

295312

296313
// ------------------------------------------------------
@@ -348,6 +365,7 @@ typedef enum mi_option_e {
348365
mi_option_abandoned_reclaim_on_free, // allow to reclaim an abandoned segment on a free (=1)
349366
mi_option_disallow_arena_alloc, // 1 = do not use arena's for allocation (except if using specific arena id's)
350367
mi_option_retry_on_oom, // retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows)
368+
mi_option_visit_abandoned, // allow visiting heap blocks from abandoned threads (=0)
351369
_mi_option_last,
352370
// legacy option names
353371
mi_option_large_os_pages = mi_option_allow_large_os_pages,

‎include/mimalloc/atomic.h

Copy file name to clipboardExpand all lines: include/mimalloc/atomic.h
+125-11Lines changed: 125 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,17 @@ terms of the MIT license. A copy of the license can be found in the file
88
#ifndef MIMALLOC_ATOMIC_H
99
#define MIMALLOC_ATOMIC_H
1010

11+
// include windows.h or pthreads.h
12+
#if defined(_WIN32)
13+
#ifndef WIN32_LEAN_AND_MEAN
14+
#define WIN32_LEAN_AND_MEAN
15+
#endif
16+
#include <windows.h>
17+
#elif !defined(__wasi__) && (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__))
18+
#define MI_USE_PTHREADS
19+
#include <pthread.h>
20+
#endif
21+
1122
// --------------------------------------------------------------------------------------------
1223
// Atomics
1324
// We need to be portable between C, C++, and MSVC.
@@ -24,9 +35,9 @@ terms of the MIT license. A copy of the license can be found in the file
2435
#define mi_atomic(name) std::atomic_##name
2536
#define mi_memory_order(name) std::memory_order_##name
2637
#if (__cplusplus >= 202002L) // c++20, see issue #571
27-
#define MI_ATOMIC_VAR_INIT(x) x
38+
#define MI_ATOMIC_VAR_INIT(x) x
2839
#elif !defined(ATOMIC_VAR_INIT)
29-
#define MI_ATOMIC_VAR_INIT(x) x
40+
#define MI_ATOMIC_VAR_INIT(x) x
3041
#else
3142
#define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x)
3243
#endif
@@ -133,10 +144,6 @@ static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) {
133144
#elif defined(_MSC_VER)
134145

135146
// Legacy MSVC plain C compilation wrapper that uses Interlocked operations to model C11 atomics.
136-
#ifndef WIN32_LEAN_AND_MEAN
137-
#define WIN32_LEAN_AND_MEAN
138-
#endif
139-
#include <windows.h>
140147
#include <intrin.h>
141148
#ifdef _WIN64
142149
typedef LONG64 msc_intptr_t;
@@ -302,11 +309,16 @@ static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub) {
302309
return (intptr_t)mi_atomic_addi(p, -sub);
303310
}
304311

312+
313+
// ----------------------------------------------------------------------
314+
// Once and Guard
315+
// ----------------------------------------------------------------------
316+
305317
typedef _Atomic(uintptr_t) mi_atomic_once_t;
306318

307319
// Returns true only on the first invocation
308320
static inline bool mi_atomic_once( mi_atomic_once_t* once ) {
309-
if (mi_atomic_load_relaxed(once) != 0) return false; // quick test
321+
if (mi_atomic_load_relaxed(once) != 0) return false; // quick test
310322
uintptr_t expected = 0;
311323
return mi_atomic_cas_strong_acq_rel(once, &expected, (uintptr_t)1); // try to set to 1
312324
}
@@ -322,17 +334,16 @@ typedef _Atomic(uintptr_t) mi_atomic_guard_t;
322334

323335

324336

337+
// ----------------------------------------------------------------------
325338
// Yield
339+
// ----------------------------------------------------------------------
340+
326341
#if defined(__cplusplus)
327342
#include <thread>
328343
static inline void mi_atomic_yield(void) {
329344
std::this_thread::yield();
330345
}
331346
#elif defined(_WIN32)
332-
#ifndef WIN32_LEAN_AND_MEAN
333-
#define WIN32_LEAN_AND_MEAN
334-
#endif
335-
#include <windows.h>
336347
static inline void mi_atomic_yield(void) {
337348
YieldProcessor();
338349
}
@@ -390,4 +401,107 @@ static inline void mi_atomic_yield(void) {
390401
#endif
391402

392403

404+
// ----------------------------------------------------------------------
405+
// Locks are only used for abandoned segment visiting in `arena.c`
406+
// ----------------------------------------------------------------------
407+
408+
#if defined(_WIN32)
409+
410+
#define mi_lock_t CRITICAL_SECTION
411+
412+
static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
413+
return TryEnterCriticalSection(lock);
414+
}
415+
static inline bool mi_lock_acquire(mi_lock_t* lock) {
416+
EnterCriticalSection(lock);
417+
return true;
418+
}
419+
static inline void mi_lock_release(mi_lock_t* lock) {
420+
LeaveCriticalSection(lock);
421+
}
422+
static inline void mi_lock_init(mi_lock_t* lock) {
423+
InitializeCriticalSection(lock);
424+
}
425+
static inline void mi_lock_done(mi_lock_t* lock) {
426+
DeleteCriticalSection(lock);
427+
}
428+
429+
430+
#elif defined(MI_USE_PTHREADS)
431+
432+
#define mi_lock_t pthread_mutex_t
433+
434+
static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
435+
return (pthread_mutex_trylock(lock) == 0);
436+
}
437+
static inline bool mi_lock_acquire(mi_lock_t* lock) {
438+
return (pthread_mutex_lock(lock) == 0);
439+
}
440+
static inline void mi_lock_release(mi_lock_t* lock) {
441+
pthread_mutex_unlock(lock);
442+
}
443+
static inline void mi_lock_init(mi_lock_t* lock) {
444+
pthread_mutex_init(lock, NULL);
445+
}
446+
static inline void mi_lock_done(mi_lock_t* lock) {
447+
pthread_mutex_destroy(lock);
448+
}
449+
450+
/*
451+
#elif defined(__cplusplus)
452+
453+
#include <mutex>
454+
#define mi_lock_t std::mutex
455+
456+
static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
457+
return lock->lock_try_acquire();
458+
}
459+
static inline bool mi_lock_acquire(mi_lock_t* lock) {
460+
lock->lock();
461+
return true;
462+
}
463+
static inline void mi_lock_release(mi_lock_t* lock) {
464+
lock->unlock();
465+
}
466+
static inline void mi_lock_init(mi_lock_t* lock) {
467+
(void)(lock);
468+
}
469+
static inline void mi_lock_done(mi_lock_t* lock) {
470+
(void)(lock);
471+
}
472+
*/
473+
474+
#else
475+
476+
// fall back to poor man's locks.
477+
// this should only be the case in a single-threaded environment (like __wasi__)
478+
479+
#define mi_lock_t _Atomic(uintptr_t)
480+
481+
static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
482+
uintptr_t expected = 0;
483+
return mi_atomic_cas_strong_acq_rel(lock, &expected, (uintptr_t)1);
484+
}
485+
static inline bool mi_lock_acquire(mi_lock_t* lock) {
486+
for (int i = 0; i < 1000; i++) { // for at most 1000 tries?
487+
if (mi_lock_try_acquire(lock)) return true;
488+
mi_atomic_yield();
489+
}
490+
return true;
491+
}
492+
static inline void mi_lock_release(mi_lock_t* lock) {
493+
mi_atomic_store_release(lock, (uintptr_t)0);
494+
}
495+
static inline void mi_lock_init(mi_lock_t* lock) {
496+
mi_lock_release(lock);
497+
}
498+
static inline void mi_lock_done(mi_lock_t* lock) {
499+
(void)(lock);
500+
}
501+
502+
#endif
503+
504+
505+
506+
393507
#endif // __MIMALLOC_ATOMIC_H

‎include/mimalloc/internal.h

Copy file name to clipboardExpand all lines: include/mimalloc/internal.h
+23-13Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -53,11 +53,6 @@ terms of the MIT license. A copy of the license can be found in the file
5353
#define mi_decl_externc
5454
#endif
5555

56-
// pthreads
57-
#if !defined(_WIN32) && !defined(__wasi__)
58-
#define MI_USE_PTHREADS
59-
#include <pthread.h>
60-
#endif
6156

6257
// "options.c"
6358
void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message);
@@ -84,11 +79,12 @@ extern mi_decl_cache_align const mi_page_t _mi_page_empty;
8479
bool _mi_is_main_thread(void);
8580
size_t _mi_current_thread_count(void);
8681
bool _mi_preloading(void); // true while the C runtime is not initialized yet
87-
mi_threadid_t _mi_thread_id(void) mi_attr_noexcept;
88-
mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap
8982
void _mi_thread_done(mi_heap_t* heap);
9083
void _mi_thread_data_collect(void);
9184
void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap);
85+
mi_threadid_t _mi_thread_id(void) mi_attr_noexcept;
86+
mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap
87+
mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id);
9288

9389
// os.c
9490
void _mi_os_init(void); // called from process init
@@ -131,15 +127,18 @@ void _mi_arena_unsafe_destroy_all(mi_stats_t* stats);
131127

132128
bool _mi_arena_segment_clear_abandoned(mi_segment_t* segment);
133129
void _mi_arena_segment_mark_abandoned(mi_segment_t* segment);
134-
size_t _mi_arena_segment_abandoned_count(void);
135130

136-
typedef struct mi_arena_field_cursor_s { // abstract
137-
mi_arena_id_t start;
138-
int count;
131+
void* _mi_arena_meta_zalloc(size_t size, mi_memid_t* memid);
132+
void _mi_arena_meta_free(void* p, mi_memid_t memid, size_t size);
133+
134+
typedef struct mi_arena_field_cursor_s { // abstract struct
135+
size_t start;
136+
size_t end;
139137
size_t bitmap_idx;
138+
mi_subproc_t* subproc;
140139
} mi_arena_field_cursor_t;
141-
void _mi_arena_field_cursor_init(mi_heap_t* heap, mi_arena_field_cursor_t* current);
142-
mi_segment_t* _mi_arena_segment_clear_abandoned_next(mi_arena_field_cursor_t* previous);
140+
void _mi_arena_field_cursor_init(mi_heap_t* heap, mi_subproc_t* subproc, mi_arena_field_cursor_t* current);
141+
mi_segment_t* _mi_arena_segment_clear_abandoned_next(mi_arena_field_cursor_t* previous, bool visit_all);
143142

144143
// "segment-map.c"
145144
void _mi_segment_map_allocated_at(const mi_segment_t* segment);
@@ -163,6 +162,7 @@ void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld);
163162
void _mi_abandoned_await_readers(void);
164163
void _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld);
165164
bool _mi_segment_attempt_reclaim(mi_heap_t* heap, mi_segment_t* segment);
165+
bool _mi_segment_visit_blocks(mi_segment_t* segment, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
166166

167167
// "page.c"
168168
void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept mi_attr_malloc;
@@ -194,6 +194,8 @@ void _mi_heap_set_default_direct(mi_heap_t* heap);
194194
bool _mi_heap_memid_is_suitable(mi_heap_t* heap, mi_memid_t memid);
195195
void _mi_heap_unsafe_destroy_all(void);
196196
mi_heap_t* _mi_heap_by_tag(mi_heap_t* heap, uint8_t tag);
197+
void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page);
198+
bool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg);
197199

198200
// "stats.c"
199201
void _mi_stats_done(mi_stats_t* stats);
@@ -349,6 +351,14 @@ static inline uintptr_t _mi_divide_up(uintptr_t size, size_t divider) {
349351
return (divider == 0 ? size : ((size + divider - 1) / divider));
350352
}
351353

354+
355+
// clamp an integer
356+
static inline size_t _mi_clamp(size_t sz, size_t min, size_t max) {
357+
if (sz < min) return min;
358+
else if (sz > max) return max;
359+
else return sz;
360+
}
361+
352362
// Is memory zero initialized?
353363
static inline bool mi_mem_is_zero(const void* p, size_t size) {
354364
for (size_t i = 0; i < size; i++) {

‎include/mimalloc/prim.h

Copy file name to clipboardExpand all lines: include/mimalloc/prim.h
+3-4Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ void _mi_prim_thread_done_auto_done(void);
115115
void _mi_prim_thread_associate_default_heap(mi_heap_t* heap);
116116

117117

118+
118119
//-------------------------------------------------------------------
119120
// Thread id: `_mi_prim_thread_id()`
120121
//
@@ -235,10 +236,6 @@ static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
235236

236237
#elif defined(_WIN32)
237238

238-
#ifndef WIN32_LEAN_AND_MEAN
239-
#define WIN32_LEAN_AND_MEAN
240-
#endif
241-
#include <windows.h>
242239
static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
243240
// Windows: works on Intel and ARM in both 32- and 64-bit
244241
return (uintptr_t)NtCurrentTeb();
@@ -370,4 +367,6 @@ static inline mi_heap_t* mi_prim_get_default_heap(void) {
370367

371368

372369

370+
371+
373372
#endif // MIMALLOC_PRIM_H

‎include/mimalloc/track.h

Copy file name to clipboardExpand all lines: include/mimalloc/track.h
+2-6Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ The corresponding `mi_track_free` still uses the block start pointer and origina
3434
The `mi_track_resize` is currently unused but could be called on reallocations within a block.
3535
`mi_track_init` is called at program start.
3636
37-
The following macros are for tools like asan and valgrind to track whether memory is
37+
The following macros are for tools like asan and valgrind to track whether memory is
3838
defined, undefined, or not accessible at all:
3939
4040
#define mi_track_mem_defined(p,size)
@@ -82,10 +82,6 @@ defined, undefined, or not accessible at all:
8282
#define MI_TRACK_HEAP_DESTROY 1
8383
#define MI_TRACK_TOOL "ETW"
8484

85-
#ifndef WIN32_LEAN_AND_MEAN
86-
#define WIN32_LEAN_AND_MEAN
87-
#endif
88-
#include <windows.h>
8985
#include "../src/prim/windows/etw.h"
9086

9187
#define mi_track_init() EventRegistermicrosoft_windows_mimalloc();
@@ -96,7 +92,7 @@ defined, undefined, or not accessible at all:
9692
// no tracking
9793

9894
#define MI_TRACK_ENABLED 0
99-
#define MI_TRACK_HEAP_DESTROY 0
95+
#define MI_TRACK_HEAP_DESTROY 0
10096
#define MI_TRACK_TOOL "none"
10197

10298
#define mi_track_malloc_size(p,reqsize,size,zero)

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.