From afa6c90df2d8f0f78f12b42a598a4e93c10a51e6 Mon Sep 17 00:00:00 2001 From: LongYinan Date: Mon, 14 Sep 2026 10:40:25 +0800 Subject: [PATCH 01/24] set _mi_process_is_initialized before mi_process_setup_auto_thread_done 439dc27b moved `mi_process_setup_auto_thread_done` into `mi_process_init_once` so it runs after `mi_thread_init` and can associate the current theap with the thread-done key. It now runs before `_mi_process_is_initialized = true`, and with `MI_TLS_RECURSE_GUARD` (always on for `MI_TLS_MODEL_LOCAL` on macOS) `_mi_theap_default()` still returns `_mi_theap_empty` at that point. The new `mi_assert_internal(mi_theap_is_initialized(theap))` fires in debug builds; release builds skip the association, so the first thread's theap is again never passed to `_mi_thread_done`. Repro (macOS): `cmake -DCMAKE_BUILD_TYPE=Debug -DMI_TLS_MODEL=LOCAL` then `./mimalloc-test-api` aborts at process init on v3.5.2 and passes on v3.5.1. Same with `-DMI_TLS_RECURSE_GUARD=ON` on Linux. Set the flag before `mi_process_setup_auto_thread_done`; the theap is fully initialized by then. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016srh5svTkqfW21zGcU6ht9 --- src/init.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/init.c b/src/init.c index 30445a58c..9b0ded5b5 100644 --- a/src/init.c +++ b/src/init.c @@ -560,8 +560,8 @@ static void mi_process_init_once(void) { // the following can potentially allocate (on freeBSD for pthread keys) _mi_tls_slots_init(); // pthread key create _mi_thread_locals_init(); // pthread key create - mi_process_setup_auto_thread_done(); // after the above mi_thread_init so it can add the current theap - _mi_process_is_initialized = true; + _mi_process_is_initialized = true; // before `mi_process_setup_auto_thread_done` so `_mi_theap_default` returns the current theap with `MI_TLS_RECURSE_GUARD` + mi_process_setup_auto_thread_done(); // after the above mi_thread_init so it can add the current theap #if defined(_WIN32) && defined(MI_WIN_INIT_USE_FLS) // On windows, when building as a static lib the FLS cleanup happens to early for the main thread. From 442ed88ad9870e027b364999ab180e28104dbcb6 Mon Sep 17 00:00:00 2001 From: Shubham Bhardwaj Date: Mon, 14 Sep 2026 19:57:15 +0530 Subject: [PATCH 02/24] fix: safely free over-aligned small allocations --- src/free.c | 14 ++++++++++---- test/test-api.c | 15 ++++++++++++++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/free.c b/src/free.c index cf352849c..e14cd1c52 100644 --- a/src/free.c +++ b/src/free.c @@ -12,6 +12,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc/prim-tls.h" // _mi_prim_thread_id() #endif + // forward declarations mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size); mi_decl_nodiscard static bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block); @@ -366,9 +367,16 @@ void mi_free_size(void* p, size_t size) mi_attr_noexcept { } void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept { - MI_UNUSED_RELEASE(alignment); mi_assert(((uintptr_t)p % alignment) == 0); - mi_free_size(p,size); + // An alignment larger than the object size is handled by over-allocation and + // can place an otherwise small object in a non-small page. In that case, + // avoid the aligned-page lookup used by mi_free_size's small fast path. + if mi_likely(alignment <= size) { + mi_free_size(p,size); + } + else { + mi_free(p); + } } void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { @@ -813,5 +821,3 @@ void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { MI_UNUSED(page); MI_UNUSED(block); } #endif - - diff --git a/test/test-api.c b/test/test-api.c index 70bba60b1..82f1a1a78 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -193,6 +193,20 @@ int main(void) { } result = ok; }; + CHECK_BODY("free-size-aligned-overaligned") { // issue #1400 + const size_t size = 8; + const size_t alignment = 16 * 1024; + void* p[200]; + bool ok = true; + for (size_t i = 0; i < 200; i++) { + p[i] = mi_malloc_aligned(size, alignment); + ok = ok && (p[i] != NULL) && ((uintptr_t)p[i] % alignment == 0); + } + for (size_t i = 0; i < 200; i++) { + mi_free_size_aligned(p[i], size, alignment); + } + result = ok; + }; CHECK_BODY("malloc-aligned5") { void* p = mi_malloc_aligned(4097,4096); size_t usable = mi_usable_size(p); @@ -725,4 +739,3 @@ static bool test_zero_aligned_first(void) { } - From 8f8e2d0714554edaaed48bb0a1e91fd578d56f12 Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 13 Sep 2026 19:33:02 -0700 Subject: [PATCH 03/24] add a counter to xthread_free to reduce reclamation rate --- include/mimalloc/internal.h | 37 ++++++++++++++++++++++++++++++++-- src/arena.c | 2 +- src/free.c | 40 +++++++++++++++++++++++-------------- src/page.c | 2 +- 4 files changed, 62 insertions(+), 19 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 80d9106a0..fb8138888 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1154,13 +1154,46 @@ static inline bool _mi_is_process_heap_main(const mi_heap_t* heap) { // Thread free flag helpers static inline mi_block_t* mi_tf_block(mi_thread_free_t tf) { + #if MI_INTPTR_BITS - MI_MAX_VABITS >= 16 + return (mi_block_t*)(((tf & ~1) << 16) >> 16); + #else return (mi_block_t*)(tf & ~1); + #endif } static inline bool mi_tf_is_owned(mi_thread_free_t tf) { return ((tf & 1) == 1); } -static inline mi_thread_free_t mi_tf_create(mi_block_t* block, bool owned) { - return (mi_thread_free_t)((uintptr_t)block | (owned ? 1 : 0)); +static inline size_t mi_tf_counter(mi_thread_free_t tf) { + #if MI_INTPTR_BITS - MI_MAX_VABITS >= 16 + return (size_t)((uintptr_t)tf >> (MI_INTPTR_BITS - 16)); + #else + return 1; + #endif +} +static inline mi_thread_free_t mi_tf_create(mi_block_t* block, bool owned, size_t counter) { + uintptr_t base = (uintptr_t)block | (owned ? 1 : 0); + #if MI_INTPTR_BITS - MI_MAX_VABITS >= 16 + mi_assert_internal(((base << 16) >> 16) == base); + base |= (uintptr_t)(counter) << (MI_INTPTR_BITS - 16); + #else + MI_UNUSED(counter); + #endif + return (mi_thread_free_t)base; +} + +// Create a new thread-free entry for the given page and block, with an appropriate counter based on the page's usage. +static inline mi_thread_free_t mi_page_tf_create(mi_page_t* page, mi_block_t* new_thread_free, bool owned) { + if (owned) { + return mi_tf_create(new_thread_free, owned, 0); + } + else { + size_t counter = mi_page_used(page); // after this many mt free's we should free the page + if (mi_page_is_full(page)) { + const uint16_t frac18 = 7 * (page->reserved / 8U); + if (frac18 < counter) { counter = (frac18 > 0 ? frac18 : 1); } // after this many mt free's we should reabandon to mapped + } + return mi_tf_create(new_thread_free, owned, counter); + } } // Thread free access diff --git a/src/arena.c b/src/arena.c index ffded1067..6d218ca10 100644 --- a/src/arena.c +++ b/src/arena.c @@ -646,7 +646,7 @@ static bool mi_abandoned_page_unown(mi_page_t* page, mi_theap_t* current_theapx) tf_old = mi_atomic_load_relaxed(&page->xthread_free); } mi_assert_internal(mi_tf_block(tf_old)==NULL); - tf_new = mi_tf_create(NULL, false); + tf_new = mi_page_tf_create(page, NULL, false); } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new)); return false; } diff --git a/src/free.c b/src/free.c index e14cd1c52..211bd79a3 100644 --- a/src/free.c +++ b/src/free.c @@ -57,7 +57,7 @@ static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool } // Forward declaration for multi-threaded collect -static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free, bool allow_reclaim) mi_attr_noexcept; +static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free, bool allow_reclaim, size_t counter) mi_attr_noexcept; // Free a block multi-threaded static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded, bool allow_reclaim) mi_attr_noexcept @@ -78,19 +78,28 @@ static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was #endif // push atomically on the page thread free list + mi_theap_t* theap = _mi_page_associated_theap_peek(page); mi_thread_free_t tf_new; mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free); do { - mi_block_set_next(page, block, mi_tf_block(tf_old)); - tf_new = mi_tf_create(block, true /* try to own it */ ); - } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new)); // todo: release is enough? + mi_block_set_next(page, block, mi_tf_block(tf_old)); + const size_t counter = mi_tf_counter(tf_old); + const bool try_reclaim = (allow_reclaim && (counter==1 || theap==page->theap)) || // always try to reclaim in our own heap + (counter==1 && !mi_tf_is_owned(tf_old)); // must try to reclaim if this is (possibly) the last block in an unowned page so we can free it + const bool new_owned = (try_reclaim ? true : mi_tf_is_owned(tf_old)); // if allow collection then always try to claim it if the page is abandoned + tf_new = mi_tf_create(block, new_owned, (counter<=1 ? counter : counter - 1)); + } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new)); // and atomically try to collect the page if it was abandoned - const bool is_owned_now = !mi_tf_is_owned(tf_old); - if (is_owned_now) { - mi_assert_internal(mi_page_is_abandoned(page)); - mi_free_try_collect_mt(page,block,allow_reclaim); - } + if (allow_reclaim) { + const bool is_newly_owned = mi_tf_is_owned(tf_new) && !mi_tf_is_owned(tf_old); + if (is_newly_owned) { + mi_assert_internal(mi_page_is_abandoned(page)); + // mi_assert_internal(!mi_page_is_abandoned_mapped(page)); + mi_free_try_collect_mt(page, block, allow_reclaim, mi_tf_counter(tf_old)); + } + } + } @@ -430,13 +439,13 @@ static bool mi_abandoned_page_try_reabandon_to_mapped(mi_page_t* page) // Release ownership of a page. This may free or reabandoned the page if other blocks are concurrently // freed in the meantime. Returns `true` if the page was freed. // By passing the captured `expected_thread_free`, we can often avoid calling `mi_page_free_collect`. -static void mi_abandoned_page_unown_from_free(mi_page_t* page, mi_block_t* expected_thread_free) { +static void mi_abandoned_page_unown_from_free(mi_page_t* page, mi_block_t* expected_thread_free, size_t old_counter) { mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(!mi_page_all_free(page)); // try to cas atomically the original free list (`mt_free`) back with the ownership cleared. - mi_thread_free_t tf_expect = mi_tf_create(expected_thread_free, true); - mi_thread_free_t tf_new = mi_tf_create(expected_thread_free, false); + mi_thread_free_t tf_expect = mi_tf_create(expected_thread_free, true, old_counter-1); + mi_thread_free_t tf_new = mi_page_tf_create(page, expected_thread_free, false); while mi_unlikely(!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_expect, tf_new)) { mi_assert_internal(mi_tf_is_owned(tf_expect)); // while the xthread_free list is not empty.. @@ -449,8 +458,9 @@ static void mi_abandoned_page_unown_from_free(mi_page_t* page, mi_block_t* expec tf_expect = mi_atomic_load_relaxed(&page->xthread_free); } // and try again to release ownership + // mi_subproc_stat_increase(mi_page_subproc(page), pages_unabandon_busy_wait, 1); mi_assert_internal(mi_tf_block(tf_expect)==NULL); - tf_new = mi_tf_create(NULL, false); + tf_new = mi_page_tf_create(page, NULL, false); } } @@ -514,7 +524,7 @@ static mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, long // We freed a block in an abandoned page (that was not owned). Try to collect -static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free, bool allow_reclaim) mi_attr_noexcept +static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free, bool allow_reclaim, size_t counter) mi_attr_noexcept { mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(mi_page_is_abandoned(page)); @@ -548,7 +558,7 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* if (mi_abandoned_page_try_reabandon_to_mapped(page)) return; // otherwise unown the page again - mi_abandoned_page_unown_from_free(page, mt_free); + mi_abandoned_page_unown_from_free(page, mt_free, counter); } diff --git a/src/page.c b/src/page.c index aa669cdd0..f26657788 100644 --- a/src/page.c +++ b/src/page.c @@ -363,7 +363,7 @@ static void mi_page_thread_free_collect(mi_page_t* page) do { head = mi_tf_block(tfree); if mi_likely(head == NULL) return; // return if the list is empty - tfreex = mi_tf_create(NULL,mi_tf_is_owned(tfree)); // set the thread free list to NULL + tfreex = mi_tf_create(NULL,mi_tf_is_owned(tfree),1); // set the thread free list to NULL } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tfree, tfreex)); // release is enough? mi_assert_internal(head != NULL); From 32a782f6dce7dfad61df94f13efdc671f8686976 Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 14 Sep 2026 10:46:28 -0700 Subject: [PATCH 04/24] update documentation for mi_free_size_aligned --- src/free.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/free.c b/src/free.c index 211bd79a3..5fbdb5124 100644 --- a/src/free.c +++ b/src/free.c @@ -344,9 +344,9 @@ void mi_free_size(void* p, size_t size) mi_attr_noexcept { const mi_page_t* const page = mi_ptr_page_validate(p,"mi_free_size"); if (page==NULL) return; mi_assert(p!=NULL); - const size_t usable = _mi_page_usable_size(page,p); + const mi_block_t* block = _mi_page_ptr_unalign(page, p); + const size_t usable = mi_page_usable_size(page,p); if mi_unlikely(size > usable) { - const mi_block_t* block = _mi_page_ptr_unalign(page, p); const bool is_guarded = mi_block_ptr_is_guarded(block,p); if (!is_guarded) { _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the size %zu is greater than the usable size %zu\n", p, size, usable); @@ -354,8 +354,8 @@ void mi_free_size(void* p, size_t size) mi_attr_noexcept { return; } } - if mi_unlikely(size <= MI_SMALL_SIZE_MAX && mi_page_block_size(page) > mi_good_size(MI_SMALL_SIZE_MAX)) { - const mi_block_t* block = _mi_page_ptr_unalign(page, p); + const size_t is_aligned = ((void*)block != p); + if mi_unlikely(size <= MI_SMALL_SIZE_MAX && mi_page_block_size(page) > mi_good_size((is_aligned ? 2 : 1)*MI_SMALL_SIZE_MAX)) { // note: we check *2 in case it was over-aligned const bool is_guarded = mi_block_ptr_is_guarded(block,p); if (!is_guarded) { _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the given size %zu is less than the allocated block size %zu\n (maybe a `new[]` was matched with `delete` instead of `delete[]`?)\n", p, size, mi_page_block_size(page)); @@ -377,9 +377,9 @@ void mi_free_size(void* p, size_t size) mi_attr_noexcept { void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept { mi_assert(((uintptr_t)p % alignment) == 0); - // An alignment larger than the object size is handled by over-allocation and - // can place an otherwise small object in a non-small page. In that case, - // avoid the aligned-page lookup used by mi_free_size's small fast path. + // If the alignment is smaller than the `size`, then for `size <= MI_SMALL_SIZE_MAX` + // the block must be allocated within a small page and can thus be handled safely by `mi_free_size`. + // (since even with over-allocation the block size will be less than 2*MI_SMALL_SIZE_MAX <= MI_SMALL_OBJ_SIZE_MAX) if mi_likely(alignment <= size) { mi_free_size(p,size); } From e8621a7e2020f104d288bd906055d9b8290e0859 Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 14 Sep 2026 17:31:51 -0700 Subject: [PATCH 05/24] clean up csize functions --- CMakeLists.txt | 4 +- include/mimalloc-new-delete.h | 2 +- include/mimalloc.h | 103 +++++++++++++++++++++------------- include/mimalloc/types.h | 7 +++ src/alloc.c | 21 ++++--- src/arena.c | 6 +- src/free.c | 2 +- test/test-api.c | 67 +++++++++++++--------- 8 files changed, 132 insertions(+), 80 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c8942709f..c1431a725 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1151,7 +1151,7 @@ if (MI_BUILD_TESTS) endforeach() # static override test - if(MI_BUILD_OBJECT AND NOT (MI_CYGWIN OR (MI_TRACK STREQUAL "ASAN") OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN)) + if(MI_BUILD_OBJECT AND MI_OVERRIDE AND NOT (MI_CYGWIN OR (MI_TRACK STREQUAL "ASAN") OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN)) add_executable(mimalloc-test-stress-static test/test-stress.c) target_compile_definitions(mimalloc-test-stress-static PRIVATE ${mi_defines} "USE_STD_MALLOC=1") target_compile_options(mimalloc-test-stress-static PRIVATE ${mi_cflags}) @@ -1161,7 +1161,7 @@ if (MI_BUILD_TESTS) endif() # dynamic override test - if(MI_BUILD_SHARED AND NOT (MI_CYGWIN OR (MI_TRACK STREQUAL "ASAN") OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN)) # AND NOT (APPLE AND MI_USE_CXX)) + if(MI_BUILD_SHARED AND MI_OVERRIDE AND NOT (MI_CYGWIN OR (MI_TRACK STREQUAL "ASAN") OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN)) # AND NOT (APPLE AND MI_USE_CXX)) add_executable(mimalloc-test-stress-dynamic test/test-stress.c) target_compile_definitions(mimalloc-test-stress-dynamic PRIVATE ${mi_defines} "USE_STD_MALLOC=1") target_compile_options(mimalloc-test-stress-dynamic PRIVATE ${mi_cflags}) diff --git a/include/mimalloc-new-delete.h b/include/mimalloc-new-delete.h index aaf185bb1..bb1bc5075 100644 --- a/include/mimalloc-new-delete.h +++ b/include/mimalloc-new-delete.h @@ -37,7 +37,7 @@ terms of the MIT license. A copy of the license can be found in the file void operator delete (void* p, const std::nothrow_t&) noexcept { mi_free(p); } void operator delete[](void* p, const std::nothrow_t&) noexcept { mi_free(p); } - mi_decl_new(n) void* operator new(std::size_t n) noexcept(false) { return mi_new(n); } + mi_decl_new(n) void* operator new(std::size_t n) noexcept(false) { return mi_new(n); } mi_decl_new(n) void* operator new[](std::size_t n) noexcept(false) { return mi_new(n); } mi_decl_new_nothrow(n) void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_new_nothrow(n); } diff --git a/include/mimalloc.h b/include/mimalloc.h index 77b2cfe2e..9a97d69ec 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -396,43 +396,6 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc_aligned( mi_decl_nodiscard mi_decl_export void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); mi_decl_nodiscard mi_decl_export void* mi_theap_rezalloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); -// ------------------------------------------------------ -// Fast constant size allocations. -// ------------------------------------------------------ - -// Machine word size allocation. `wsize` is the allocation size in machine words (`sizeof(size_t)`) -mi_decl_nodiscard mi_decl_restrict void* mi_wzalloc_small(size_t wsize) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_restrict void* mi_wmalloc_small(size_t wsize) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_restrict void* mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_restrict void* mi_theap_wzalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept; - -// get the machine word size from a byte size. -static inline size_t mi_wsize_from_size(size_t size) { - return ((size + sizeof(size_t) - 1) / sizeof(size_t)); -} - -static inline mi_decl_restrict void* mi_malloc_csize(size_t size) mi_attr_noexcept { - if (size <= MI_SMALL_SIZE_MAX) { return mi_wmalloc_small(mi_wsize_from_size(size)); } else { return mi_malloc(size); } -} -static inline mi_decl_restrict void* mi_zalloc_csize(size_t size) mi_attr_noexcept { - if (size <= MI_SMALL_SIZE_MAX) { return mi_wzalloc_small(mi_wsize_from_size(size)); } else { return mi_zalloc(size); } -} -static inline mi_decl_restrict void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { - assert(theap!=NULL); - if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_wmalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_malloc(theap,size); } -} -static inline mi_decl_restrict void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { - assert(theap!=NULL); - if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_wzalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_zalloc(theap,size); } -} -static inline void mi_free_csize(void* p, size_t size) mi_attr_noexcept { - if (size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } else { mi_free(p); } -} -static inline void mi_free_csize_nonnull(void* p, size_t size) mi_attr_noexcept { - assert(p!=NULL); - if (size <= MI_SMALL_SIZE_MAX) { mi_free_small_nonnull(p); } else { mi_free(p); } -} - // ------------------------------------------------------ // Experimental // ------------------------------------------------------ @@ -588,8 +551,13 @@ mi_decl_export int mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name mi_decl_nodiscard mi_decl_export mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s) mi_attr_noexcept mi_attr_malloc; mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept mi_attr_malloc; -// The `mi_new` wrappers implement C++ semantics on out-of-memory instead of directly returning `NULL`. -// (and call `std::get_new_handler` and potentially raise a `std::bad_alloc` exception). +// -------------------------------------------------------- +// C++ wrappers +// The `mi_new` wrappers implement C++ semantics on out-of-memory +// instead of directly returning `NULL`. (and call `std::get_new_handler` +// and potentially raise a `std::bad_alloc` exception). +// -------------------------------------------------------- + mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new(size_t size) mi_attr_malloc mi_attr_alloc_size(1); mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); @@ -601,6 +569,63 @@ mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2); mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* theap, size_t size) mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* theap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_alloc_new_nothrow(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); + + +// -------------------------------------------------------------------------- +// Inlined constant size allocations. +// These are meant for runtime systems, or overrides where we need the best +// performance for small, constant-size allocations. +// These are only better than the regular functions if the size or alignment +// are indeed constant at the call site. +// -------------------------------------------------------------------------- + +// Internal machine word size allocation. `wsize` is the allocation size in machine words (`sizeof(size_t)`) +mi_decl_nodiscard mi_decl_restrict void* mi_wzalloc_small(size_t wsize) mi_attr_noexcept mi_attr_malloc; +mi_decl_nodiscard mi_decl_restrict void* mi_wmalloc_small(size_t wsize) mi_attr_noexcept mi_attr_malloc; +mi_decl_nodiscard mi_decl_restrict void* mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept mi_attr_malloc; +mi_decl_nodiscard mi_decl_restrict void* mi_theap_wzalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept mi_attr_malloc; + +static inline size_t mi_wsize_from_size(size_t size) mi_attr_noexcept { + return (size + sizeof(size_t) - 1) / sizeof(size_t); +} + +static inline mi_decl_restrict void* mi_malloc_csize(size_t size) mi_attr_noexcept { + if (size <= MI_SMALL_SIZE_MAX) { return mi_wmalloc_small(mi_wsize_from_size(size)); } else { return mi_malloc(size); } +} +static inline mi_decl_restrict void* mi_zalloc_csize(size_t size) mi_attr_noexcept { + if (size <= MI_SMALL_SIZE_MAX) { return mi_wzalloc_small(mi_wsize_from_size(size)); } else { return mi_zalloc(size); } +} +static inline mi_decl_restrict void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { + assert(theap!=NULL); + if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_wmalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_malloc(theap,size); } +} +static inline mi_decl_restrict void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { + assert(theap!=NULL); + if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_wzalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_zalloc(theap,size); } +} + +static inline void mi_free_csize(void* p, size_t size) mi_attr_noexcept { + if (size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } else { mi_free(p); } +} +static inline void mi_free_csize_nonnull(void* p, size_t size) mi_attr_noexcept { + assert(p!=NULL); + if (size <= MI_SMALL_SIZE_MAX) { mi_free_small_nonnull(p); } else { mi_free(p); } +} +static inline void mi_free_csize_aligned(void* p, size_t size, size_t aligned) mi_attr_noexcept { + if (aligned <= size && size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } else { mi_free(p); } +} +static inline void mi_free_csize_aligned_nonnull(void* p, size_t size, size_t aligned) mi_attr_noexcept { + assert(p!=NULL); + if (aligned <= size && size <= MI_SMALL_SIZE_MAX) { mi_free_small_nonnull(p); } else { mi_free(p); } +} + +// ------------------------------------------------------ +// C++ standard library allocator interface. +// ------------------------------------------------------ + #ifdef __cplusplus } #endif diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 21e787b43..530c464a3 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -535,6 +535,13 @@ typedef struct mi_page_s { #define MI_MAX_SINGLETON_BIN MI_BIN_HUGE #endif +// The small object max size must be larger than twice the small size max minus one, +// such that any aligned allocation with `alignment <= size <= MI_SMALL_SIZE_MAX` will +// be allocated in a small page (so `mi_free_csize` can delegate correctly to `mi_free_small`. +#if MI_SMALL_MAX_OBJ_SIZE <= (2*(MI_SMALL_WSIZE_MAX * MI_SIZE_SIZE)-1) +#error "mimalloc internal: the small object max size is too small" +#endif + // ------------------------------------------------------ // Page kinds // ------------------------------------------------------ diff --git a/src/alloc.c b/src/alloc.c index ded0a00c5..8b8fc40cb 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -818,15 +818,15 @@ static mi_decl_noinline void* mi_theap_try_new(mi_theap_t* theap, size_t size, b return p; } -static mi_decl_noinline void* mi_try_new(size_t size, bool nothrow) { - return mi_theap_try_new(_mi_theap_default(), size, nothrow); -} +// static mi_decl_noinline void* mi_try_new(size_t size, bool nothrow) { +// return mi_theap_try_new(_mi_theap_default(), size, nothrow); +// } static mi_decl_noinline void* mi_heap_try_new(mi_heap_t* heap, size_t size, bool nothrow) { return mi_theap_try_new(_mi_heap_theap(heap), size, nothrow); } -mi_decl_nodiscard static mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* theap, size_t size) { +mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* theap, size_t size) { void* p = mi_theap_malloc(theap,size); if mi_unlikely(p == NULL) return mi_theap_try_new(theap, size, false); return p; @@ -842,7 +842,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size return p; } -mi_decl_nodiscard static mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* theap, size_t count, size_t size) { +mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* theap, size_t count, size_t size) { size_t total; if mi_unlikely(mi_count_size_overflow(count, size, &total)) { mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc @@ -861,12 +861,17 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, si return mi_theap_alloc_new_n(_mi_heap_theap(heap), count, size); } -mi_decl_nodiscard mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept { - void* p = mi_malloc(size); - if mi_unlikely(p == NULL) return mi_try_new(size, true); + +mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new_nothrow(mi_theap_t* theap, size_t size) mi_attr_noexcept { + void* p = mi_theap_malloc(theap,size); + if mi_unlikely(p == NULL) return mi_theap_try_new(theap, size, true); return p; } +mi_decl_nodiscard mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept { + return mi_theap_alloc_new_nothrow(_mi_theap_default(), size); +} + static mi_decl_noinline void* mi_try_new_aligned(size_t size, size_t alignment, bool nothrow) { void* p = NULL; for(int i = 0; i < MI_TRY_NEW_MAX && p==NULL && mi_try_new_handler(nothrow); i++) { diff --git a/src/arena.c b/src/arena.c index 6d218ca10..7e45ab8c8 100644 --- a/src/arena.c +++ b/src/arena.c @@ -889,11 +889,15 @@ mi_decl_maybe_unused static size_t mi_page_block_start(size_t block_size, bool o if (os_align) { offset = MI_PAGE_ALIGN; } - else if (_mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) { + else if (block_size != 0 && _mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) { // naturally align power-of-2 blocks up to MI_PAGE_MAX_START_BLOCK_ALIGN2 size (4KiB) offset = _mi_align_up(mi_page_info_size(), block_size); if (block_size < 64) { offset += 3*block_size; } } + else if (block_size != 0 && block_size <= MI_SMALL_SIZE_MAX) { + // align small blocks to their size + offset = _mi_align_up(mi_page_info_size(), block_size); + } else if (block_size != 0 && (block_size % MI_PAGE_OSPAGE_BLOCK_ALIGN2) == 0) { // also align large pages that are a multiple of MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4KiB) offset = _mi_align_up(mi_page_info_size(), MI_PAGE_OSPAGE_BLOCK_ALIGN2); diff --git a/src/free.c b/src/free.c index 5fbdb5124..223831dd4 100644 --- a/src/free.c +++ b/src/free.c @@ -345,7 +345,7 @@ void mi_free_size(void* p, size_t size) mi_attr_noexcept { if (page==NULL) return; mi_assert(p!=NULL); const mi_block_t* block = _mi_page_ptr_unalign(page, p); - const size_t usable = mi_page_usable_size(page,p); + const size_t usable = _mi_page_usable_size(page,p); if mi_unlikely(size > usable) { const bool is_guarded = mi_block_ptr_is_guarded(block,p); if (!is_guarded) { diff --git a/test/test-api.c b/test/test-api.c index 82f1a1a78..1f6255926 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -73,26 +73,6 @@ static bool mem_is_zero(const void* p, size_t size) { int main(void) { mi_option_disable(mi_option_verbose); - #if 1 - #if defined(__cplusplus) && !defined(_MSC_VER) - CHECK_BODY("c++ new-handler") { - std::set_new_handler([]{ throw std::bad_alloc(); }); - void* p = mi_new_nothrow(SIZE_MAX/2); - result = (p==NULL); - } - CHECK_BODY("c++ new handler2") { - try { - void* p = mi_new_n(SIZE_MAX/2, 4); - (void)(p); - result = false; - } - catch(std::bad_alloc) { - result = true; - } - } - #endif - #endif - // --------------------------------------------------- // Malloc // --------------------------------------------------- @@ -319,12 +299,22 @@ int main(void) { for(int i = 0; i < 10 && ok; i++) { mi_free(p[i]); } - /* - if (ok && align <= size && ((size + MI_PADDING_SIZE) & (align-1)) == 0) { - size_t bsize = mi_good_size(size); - ok = (align <= bsize && (bsize & (align-1)) == 0); + } + } + result = ok; + } + CHECK_BODY("mimalloc-size-aligned14") { + bool ok = true; + for( size_t size = 1; size <= (MI_SMALL_SIZE_MAX * 2) && ok; size++ ) { + for(size_t align = 1; align <= 16*size && ok; align *= 2) { + void* p[10]; + for(int i = 0; i < 10 && ok; i++) { + p[i] = mi_malloc_aligned(size,align);; + ok = (p[i] != NULL && ((uintptr_t)(p[i]) % align) == 0); + } + for(int i = 0; i < 10 && ok; i++) { + mi_free_size_aligned(p[i],size,align); } - */ } } result = ok; @@ -355,9 +345,7 @@ int main(void) { memset(junk, 0xAB, size); mi_free(junk); uint8_t* z = (uint8_t*)mi_theap_zalloc_csize(theap, size); - for (size_t i = 0; i < size; i++) { - if (z[i] != 0) { ok = false; break; } - } + ok = mem_is_zero(z, size); mi_free(z); } result = ok; @@ -496,6 +484,29 @@ int main(void) { } #endif + + // --------------------------------------------------- + // C++ + // --------------------------------------------------- + + #if defined(__cplusplus) && !defined(_MSC_VER) + CHECK_BODY("c++ new-handler") { + std::set_new_handler([]{ throw std::bad_alloc(); }); + void* p = mi_new_nothrow(SIZE_MAX/2); + result = (p==NULL); + } + CHECK_BODY("c++ new handler2") { + try { + void* p = mi_new_n(SIZE_MAX/2, 4); + (void)(p); + result = false; + } + catch(std::bad_alloc) { + result = true; + } + } + #endif + // --------------------------------------------------- // Heaps // --------------------------------------------------- From 8bf78206aa0ba83c27ff9b48c14b6f9699886ec6 Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 14 Sep 2026 21:24:04 -0700 Subject: [PATCH 06/24] revise handling of NULL theaps using xtheap naming convention; fixes issue #1398 --- include/mimalloc/internal.h | 9 +- src/alloc-aligned.c | 136 +++++++++----------- src/alloc.c | 239 +++++++++++++++++++----------------- test/test-api.c | 20 ++- 4 files changed, 211 insertions(+), 193 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index fb8138888..44aa92f20 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -270,8 +270,8 @@ mi_page_t* _mi_safe_ptr_page(const void* p); void _mi_page_map_unsafe_destroy(void); // "page.c" -void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; -void* _mi_malloc_generic_no_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; +void* _mi_malloc_generic(mi_theap_t* xtheap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; +void* _mi_malloc_generic_no_sample(mi_theap_t* xtheap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks void _mi_page_unfull(mi_page_t* page); @@ -323,8 +323,8 @@ mi_msecs_t _mi_clock_start(void); // "alloc.c" void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` -void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` -void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero) mi_attr_noexcept; +void* _mi_xtheap_malloc_zero(mi_theap_t* xtheap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` +void* _mi_xtheap_realloc_zero(mi_theap_t* xtheap, void* p, size_t newsize, bool zero) mi_attr_noexcept; mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p); void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size); @@ -654,6 +654,7 @@ extern mi_decl_hidden mi_theap_t _mi_theap_empty_wrong; // read-only empty theap static inline mi_heap_t* _mi_theap_heap_peek(const mi_theap_t* theap) { + mi_assert_internal(theap!=NULL); return mi_atomic_load_ptr_relaxed(mi_heap_t,&theap->heap); } diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 3a4383e0b..bdd7f626b 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -48,26 +48,10 @@ static mi_decl_noinline mi_decl_restrict void* mi_theap_malloc_guarded_aligned(m return p; } #endif -#if MI_GUARDED -static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) { - // #if MI_THEAP_INITASNULL - // if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); } - // #endif - // const size_t rate = theap->guarded_sample_rate; - // only write if `rate!=0` so we don't write to the constant `_mi_theap_empty` - // if (rate != 0) { theap->guarded_sample_rate = 0; } - void* p = _mi_theap_malloc_zero(theap, size, zero, 0, ppage); - // if (rate != 0) { theap->guarded_sample_rate = rate; } - return p; -} -#else -static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) { - return _mi_theap_malloc_zero(theap, size, zero, 0, ppage); -} -#endif + // Fallback aligned allocation that over-allocates -- split out for better codegen -static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept +static mi_decl_noinline void* mi_xtheap_malloc_zero_aligned_at_overalloc(mi_theap_t* const xtheap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept { mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)); mi_assert_internal(mi_alignment_is_valid(alignment)); @@ -86,7 +70,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap } oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size); // note: no guarded as alignment > 0 - p = _mi_theap_malloc_zero(theap, oversize, zero, alignment, &page); // the page block size should be large enough to align in the single huge page block + p = _mi_xtheap_malloc_zero(xtheap, oversize, zero, alignment, &page); // the page block size should be large enough to align in the single huge page block if (p == NULL) return NULL; } else { @@ -94,7 +78,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) && alignment <= MI_PAGE_MAX_OVERALLOC_ALIGN); mi_assert_internal(size < SIZE_MAX - alignment); // `oversize` cannot overflow oversize = (size < MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : size) + alignment - 1; // adjust for size <= 16; with size 0 and alignment 64k, we would allocate a 64k block and pointing just beyond that. - p = mi_theap_malloc_zero_no_guarded(theap, oversize, zero, &page); + p = _mi_xtheap_malloc_zero(xtheap, oversize, zero, 0, &page); if (p == NULL) return NULL; } mi_assert_internal(page == _mi_ptr_page(p)); @@ -148,7 +132,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap } // Generic primitive aligned allocation -- split out for better codegen -static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept +static mi_decl_noinline void* mi_xtheap_malloc_zero_aligned_at_generic(mi_theap_t* const xtheap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept { mi_assert_internal(mi_alignment_is_valid(alignment)); // we don't allocate more than MI_MAX_ALLOC_SIZE (see ) @@ -161,17 +145,17 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t // this is important to try as the fast path in `mi_theap_malloc_zero_aligned` only works when there exist // a page with the right block size, and if we always use the over-alloc fallback that would never happen. #if MI_THEAP_INITASNULL - if mi_likely(theap!=NULL) + if mi_likely(xtheap!=NULL) #endif { #if MI_SAMPLE // only try if we would not take a sample - if mi_likely(!mi_theap_should_sample(theap,size)) + if mi_likely(!mi_theap_should_sample(xtheap,size)) #endif { if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) { mi_page_t* page = NULL; - void* p = mi_theap_malloc_zero_no_guarded(theap, size, zero, &page); + void* p = _mi_xtheap_malloc_zero(xtheap, size, zero, 0,&page); if (ppage!=NULL) { *ppage = page; } const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0; if mi_likely(is_aligned_or_null) { @@ -188,7 +172,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t } // fall back to over-allocation - return mi_theap_malloc_zero_aligned_at_overalloc(theap,size,alignment,offset,zero,ppage); + return mi_xtheap_malloc_zero_aligned_at_overalloc(xtheap,size,alignment,offset,zero,ppage); } @@ -198,7 +182,7 @@ static mi_decl_cold mi_decl_noinline void* mi_error_bad_alignment(size_t size, s } // Primitive aligned allocation -static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept +static inline void* mi_xtheap_malloc_zero_aligned_at(mi_theap_t* const xtheap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept { // note: we don't require `size > offset`, we just guarantee that the address at offset is aligned regardless of the allocated size. if mi_unlikely(!mi_alignment_is_valid(alignment)) { // require power-of-two and multiple of void* (see ) @@ -211,22 +195,22 @@ static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, con if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size) { #if MI_THEAP_INITASNULL - if mi_likely(theap!=NULL) + if mi_likely(xtheap!=NULL) #endif { #if MI_SAMPLE // check if we shouldn't take a sample - if mi_likely(!mi_theap_should_sample(theap,size)) + if mi_likely(!mi_theap_should_sample(xtheap,size)) #endif { const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` const size_t padsize = size + MI_PADDING_SIZE; - mi_page_t* page = _mi_theap_get_free_small_page(theap, padsize, false); + mi_page_t* page = _mi_theap_get_free_small_page(xtheap, padsize, false); if mi_likely(page->free != NULL) { const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0; if mi_likely(is_aligned) { if (ppage!=NULL) { *ppage = page; } - void* p = _mi_page_malloc_zero(theap, page, padsize, zero); + void* p = _mi_page_malloc_zero(xtheap, page, padsize, zero); // xtheap!=NULL mi_assert_internal(p != NULL); mi_assert_internal(((uintptr_t)p + offset) % alignment == 0); mi_track_malloc(p, size, zero); @@ -238,7 +222,7 @@ static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, con } // fallback to generic aligned allocation - return mi_theap_malloc_zero_aligned_at_generic(theap, size, alignment, offset, zero, ppage); + return mi_xtheap_malloc_zero_aligned_at_generic(xtheap, size, alignment, offset, zero, ppage); } @@ -246,30 +230,30 @@ static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, con // Internal mi_theap_malloc_aligned / mi_malloc_aligned // ------------------------------------------------------ -static mi_decl_restrict void* mi_theap_malloc_aligned_at(mi_theap_t* theap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_theap_malloc_zero_aligned_at(theap, size, alignment, offset, false, NULL); +static mi_decl_restrict void* mi_theap_malloc_aligned_at(mi_theap_t* xtheap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_xtheap_malloc_zero_aligned_at(xtheap, size, alignment, offset, false, NULL); } -mi_decl_nodiscard mi_decl_restrict void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept { - return mi_theap_malloc_aligned_at(theap, size, alignment, 0); +mi_decl_nodiscard mi_decl_restrict void* mi_theap_malloc_aligned(mi_theap_t* xtheap, size_t size, size_t alignment) mi_attr_noexcept { + return mi_theap_malloc_aligned_at(xtheap, size, alignment, 0); } -static mi_decl_restrict void* mi_theap_zalloc_aligned_at(mi_theap_t* theap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_theap_malloc_zero_aligned_at(theap, size, alignment, offset, true, NULL); +static mi_decl_restrict void* mi_theap_zalloc_aligned_at(mi_theap_t* xtheap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_xtheap_malloc_zero_aligned_at(xtheap, size, alignment, offset, true, NULL); } -mi_decl_restrict void* mi_theap_zalloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept { - return mi_theap_zalloc_aligned_at(theap, size, alignment, 0); +mi_decl_restrict void* mi_theap_zalloc_aligned(mi_theap_t* xtheap, size_t size, size_t alignment) mi_attr_noexcept { + return mi_theap_zalloc_aligned_at(xtheap, size, alignment, 0); } -static mi_decl_restrict void* mi_theap_calloc_aligned_at(mi_theap_t* theap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { +static mi_decl_restrict void* mi_theap_calloc_aligned_at(mi_theap_t* xtheap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { size_t total; if (mi_count_size_overflow(count, size, &total)) return NULL; - return mi_theap_zalloc_aligned_at(theap, total, alignment, offset); + return mi_theap_zalloc_aligned_at(xtheap, total, alignment, offset); } -static mi_decl_restrict void* mi_theap_calloc_aligned(mi_theap_t* theap, size_t count, size_t size, size_t alignment) mi_attr_noexcept { - return mi_theap_calloc_aligned_at(theap, count, size, alignment, 0); +static mi_decl_restrict void* mi_theap_calloc_aligned(mi_theap_t* xtheap, size_t count, size_t size, size_t alignment) mi_attr_noexcept { + return mi_theap_calloc_aligned_at(xtheap, count, size, alignment, 0); } @@ -287,7 +271,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t a mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, false, &page); + void* p = mi_xtheap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, false, &page); if (p!=NULL && pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); } return p; } @@ -302,7 +286,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t a mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, true, &page); + void* p = mi_xtheap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, true, &page); if (p!=NULL && pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); } return p; } @@ -345,13 +329,13 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, // Aligned re-allocation // ------------------------------------------------------ -static void* mi_theap_realloc_zero_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero) mi_attr_noexcept { +static void* mi_xtheap_realloc_zero_aligned_at(mi_theap_t* xtheap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero) mi_attr_noexcept { mi_assert(mi_alignment_is_valid(alignment)); if mi_unlikely(!mi_alignment_is_valid(alignment)) { // require power-of-two (see ) return mi_error_bad_alignment(newsize,alignment,offset); } - if (alignment <= sizeof(uintptr_t) && offset==0) return _mi_theap_realloc_zero(theap,p,newsize,zero); - if (p == NULL) return mi_theap_malloc_zero_aligned_at(theap,newsize,alignment,offset,zero,NULL); + if (alignment <= sizeof(uintptr_t) && offset==0) return _mi_xtheap_realloc_zero(xtheap,p,newsize,zero); + if (p == NULL) return mi_xtheap_malloc_zero_aligned_at(xtheap,newsize,alignment,offset,zero,NULL); const size_t size = mi_usable_size(p); if (newsize <= size && newsize >= (size - (size / 2)) && (((uintptr_t)p + offset) & (alignment-1)) == 0) { return p; // reallocation still fits, is aligned and not more than 50% waste @@ -359,7 +343,7 @@ static void* mi_theap_realloc_zero_aligned_at(mi_theap_t* theap, void* p, size_t else { // note: we don't zero allocate upfront so we only zero initialize the expanded part (at the cost of calling mi_usable_size) mi_page_t* newpage; - void* const newp = mi_theap_malloc_zero_aligned_at(theap,newsize,alignment,offset,false/*zero?*/,&newpage); + void* const newp = mi_xtheap_malloc_zero_aligned_at(xtheap,newsize,alignment,offset,false/*zero?*/,&newpage); if (newp != NULL) { const size_t copy_size = (newsize > size ? size : newsize); const size_t zero_start = (copy_size >= sizeof(intptr_t) ? copy_size - sizeof(intptr_t) : 0); // also set last word in the previous allocation to zero to ensure any padding is zero-initialized @@ -376,88 +360,88 @@ static void* mi_theap_realloc_zero_aligned_at(mi_theap_t* theap, void* p, size_t } } -static void* mi_theap_realloc_zero_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, bool zero) mi_attr_noexcept { +static void* mi_xtheap_realloc_zero_aligned(mi_theap_t* xtheap, void* p, size_t newsize, size_t alignment, bool zero) mi_attr_noexcept { mi_assert(alignment > 0); - if (alignment <= sizeof(uintptr_t)) return _mi_theap_realloc_zero(theap,p,newsize,zero); - return mi_theap_realloc_zero_aligned_at(theap,p,newsize,alignment,0,zero); + if (alignment <= sizeof(uintptr_t)) return _mi_xtheap_realloc_zero(xtheap,p,newsize,zero); + return mi_xtheap_realloc_zero_aligned_at(xtheap,p,newsize,alignment,0,zero); } -static void* mi_theap_realloc_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_theap_realloc_zero_aligned_at(theap,p,newsize,alignment,offset,false); +static void* mi_xtheap_realloc_aligned_at(mi_theap_t* xtheap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_xtheap_realloc_zero_aligned_at(xtheap,p,newsize,alignment,offset,false); } -static void* mi_theap_realloc_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { - return mi_theap_realloc_zero_aligned(theap,p,newsize,alignment,false); +static void* mi_xtheap_realloc_aligned(mi_theap_t* xtheap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { + return mi_xtheap_realloc_zero_aligned(xtheap,p,newsize,alignment,false); } -static void* mi_theap_rezalloc_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_theap_realloc_zero_aligned_at(theap, p, newsize, alignment, offset, true); +static void* mi_xtheap_rezalloc_aligned_at(mi_theap_t* xtheap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_xtheap_realloc_zero_aligned_at(xtheap, p, newsize, alignment, offset, true); } -static void* mi_theap_rezalloc_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { - return mi_theap_realloc_zero_aligned(theap, p, newsize, alignment, true); +static void* mi_xtheap_rezalloc_aligned(mi_theap_t* xtheap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { + return mi_xtheap_realloc_zero_aligned(xtheap, p, newsize, alignment, true); } -static void* mi_theap_recalloc_aligned_at(mi_theap_t* theap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { +static void* mi_xtheap_recalloc_aligned_at(mi_theap_t* xtheap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { size_t total; if (mi_count_size_overflow(newcount, size, &total)) return NULL; - return mi_theap_rezalloc_aligned_at(theap, p, total, alignment, offset); + return mi_xtheap_rezalloc_aligned_at(xtheap, p, total, alignment, offset); } -static void* mi_theap_recalloc_aligned(mi_theap_t* theap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { +static void* mi_xtheap_recalloc_aligned(mi_theap_t* xtheap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { size_t total; if (mi_count_size_overflow(newcount, size, &total)) return NULL; - return mi_theap_rezalloc_aligned(theap, p, total, alignment); + return mi_xtheap_rezalloc_aligned(xtheap, p, total, alignment); } mi_decl_nodiscard void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_theap_realloc_aligned_at(_mi_theap_default(), p, newsize, alignment, offset); + return mi_xtheap_realloc_aligned_at(_mi_theap_default(), p, newsize, alignment, offset); } mi_decl_nodiscard void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept { - return mi_theap_realloc_aligned(_mi_theap_default(), p, newsize, alignment); + return mi_xtheap_realloc_aligned(_mi_theap_default(), p, newsize, alignment); } mi_decl_nodiscard void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_theap_rezalloc_aligned_at(_mi_theap_default(), p, newsize, alignment, offset); + return mi_xtheap_rezalloc_aligned_at(_mi_theap_default(), p, newsize, alignment, offset); } mi_decl_nodiscard void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept { - return mi_theap_rezalloc_aligned(_mi_theap_default(), p, newsize, alignment); + return mi_xtheap_rezalloc_aligned(_mi_theap_default(), p, newsize, alignment); } mi_decl_nodiscard void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_theap_recalloc_aligned_at(_mi_theap_default(), p, newcount, size, alignment, offset); + return mi_xtheap_recalloc_aligned_at(_mi_theap_default(), p, newcount, size, alignment, offset); } mi_decl_nodiscard void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { - return mi_theap_recalloc_aligned(_mi_theap_default(), p, newcount, size, alignment); + return mi_xtheap_recalloc_aligned(_mi_theap_default(), p, newcount, size, alignment); } mi_decl_nodiscard void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_theap_realloc_aligned_at(_mi_heap_theap(heap), p, newsize, alignment, offset); + return mi_xtheap_realloc_aligned_at(_mi_heap_theap(heap), p, newsize, alignment, offset); } mi_decl_nodiscard void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { - return mi_theap_realloc_aligned(_mi_heap_theap(heap), p, newsize, alignment); + return mi_xtheap_realloc_aligned(_mi_heap_theap(heap), p, newsize, alignment); } mi_decl_nodiscard void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_theap_rezalloc_aligned_at(_mi_heap_theap(heap), p, newsize, alignment, offset); + return mi_xtheap_rezalloc_aligned_at(_mi_heap_theap(heap), p, newsize, alignment, offset); } mi_decl_nodiscard void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { - return mi_theap_rezalloc_aligned(_mi_heap_theap(heap), p, newsize, alignment); + return mi_xtheap_rezalloc_aligned(_mi_heap_theap(heap), p, newsize, alignment); } mi_decl_nodiscard void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_theap_recalloc_aligned_at(_mi_heap_theap(heap), p, newcount, size, alignment, offset); + return mi_xtheap_recalloc_aligned_at(_mi_heap_theap(heap), p, newcount, size, alignment, offset); } mi_decl_nodiscard void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { - return mi_theap_recalloc_aligned(_mi_heap_theap(heap), p, newcount, size, alignment); + return mi_xtheap_recalloc_aligned(_mi_heap_theap(heap), p, newcount, size, alignment); } diff --git a/src/alloc.c b/src/alloc.c index 8b8fc40cb..e81bfa36a 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -1,4 +1,3 @@ - /* ---------------------------------------------------------------------------- Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the @@ -145,7 +144,7 @@ extern void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t siz } // internal small size allocation -static mi_decl_forceinline mi_decl_restrict void* mi_theap_xmalloc_small_zero_nonnull(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept +static mi_decl_forceinline mi_decl_restrict void* mi_theap_xmalloc_small_zero(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept { size_t size = (is_wsize ? xsize * MI_SIZE_SIZE : xsize); mi_assert(theap != NULL); @@ -178,14 +177,15 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_xmalloc_small_zero_no } // internal generic allocation -static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept +// we use mi_xtheap_ to signify that the `theap` can be NULL (on Windows and macOS) +static mi_decl_forceinline void* mi_xtheap_malloc_generic(mi_theap_t* xtheap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { #if !MI_THEAP_INITASNULL - mi_assert(theap!=NULL); + mi_assert(xtheap!=NULL); #endif - mi_assert(mi_theap_matches_thread(theap)); // theaps are thread local + mi_assert(mi_theap_matches_thread(xtheap)); // theaps are thread local mi_assert((huge_alignment & 1)==0); - void* const p = _mi_malloc_generic(theap, size + MI_PADDING_SIZE, (zero ? 1 : 0) | huge_alignment, ppage); // note: size can overflow but it is detected in malloc_generic + void* const p = _mi_malloc_generic(xtheap, size + MI_PADDING_SIZE, (zero ? 1 : 0) | huge_alignment, ppage); // note: size can overflow but it is detected in malloc_generic mi_track_malloc(p, size, zero); #if MI_DEBUG>3 @@ -196,77 +196,89 @@ static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size return p; } + +// ------------------------------------------------------------------------ +// We make extensive use of inline functions to define all allocation +// variants. The number of variants is a bit large as we like to optimize them well. +// +// For example, the `theap` variants should not have NULL +// theap pointers so we can avoid a NULL check. We use the `xtheap` +// name for theaps that can potentially be NULL. This is perhaps a bit too +// much though as it is only needed for Windows and macOS. +// We already simplify this for realloc and new variants. +// ------------------------------------------------------------------------ + // internal small allocation -static mi_decl_forceinline mi_decl_restrict void* mi_theap_xmalloc_small_zero(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept { +static mi_decl_forceinline mi_decl_restrict void* mi_xtheap_xmalloc_small_zero(mi_theap_t* xtheap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept { #if MI_THEAP_INITASNULL - if (theap!=NULL) { - return mi_theap_xmalloc_small_zero_nonnull(theap, xsize, is_wsize, zero, ppage); + if (xtheap!=NULL) { + return mi_theap_xmalloc_small_zero(xtheap, xsize, is_wsize, zero, ppage); } else { - return mi_theap_malloc_generic(theap, (is_wsize ? xsize * MI_SIZE_SIZE : xsize), zero, 0, ppage); // tailcall + return mi_xtheap_malloc_generic(xtheap, (is_wsize ? xsize * MI_SIZE_SIZE : xsize), zero, 0, ppage); // tailcall } #else - return mi_theap_xmalloc_small_zero_nonnull(theap, xsize, is_wsize, zero, ppage); + return mi_xtheap_xmalloc_small_zero(xtheap, xsize, is_wsize, zero, ppage); #endif } // allocate a small block mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { mi_assert(theap!=NULL); - return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, false, NULL); + return mi_theap_xmalloc_small_zero(theap, size, false, false, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept { mi_assert(theap!=NULL); - return mi_theap_xmalloc_small_zero_nonnull(theap, wsize, true, false, NULL); + return mi_theap_xmalloc_small_zero(theap, wsize, true, false, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept { - return mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, false, NULL); + return mi_xtheap_xmalloc_small_zero(_mi_theap_default(), size, false, false, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_wmalloc_small(size_t wsize) mi_attr_noexcept { - return mi_theap_xmalloc_small_zero(_mi_theap_default(), wsize, true, false, NULL ); + return mi_xtheap_xmalloc_small_zero(_mi_theap_default(), wsize, true, false, NULL ); } mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_xmalloc_small_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, false, NULL); + return mi_theap_malloc_small(_mi_heap_theap_get_or_init(heap), size); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { // we could also use: mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(theap), size, false, NULL); } // but the following prevents using a stack frame. We use this to optimize some select functions only. - mi_theap_t* const theap = _mi_heap_theap_cached(heap); - if mi_likely(theap!=NULL) { return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, false, NULL); } - else { return mi_heap_init_malloc_small(heap, size); } + mi_theap_t* const xtheap = _mi_heap_theap_cached(heap); + if mi_likely(xtheap!=NULL) { return mi_theap_malloc_small(xtheap, size); } + else { return mi_heap_init_malloc_small(heap, size); } } // The main internal allocation functions -static mi_decl_forceinline void* mi_theap_malloc_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { +static mi_decl_forceinline void* mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { mi_assert(theap!=NULL); // fast path for small objects if mi_likely(size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(huge_alignment == 0); - return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, zero, ppage); + return mi_theap_xmalloc_small_zero(theap, size, false, zero, ppage); } else { - return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage); + return mi_xtheap_malloc_generic(theap, size, zero, huge_alignment, ppage); } } -extern mi_decl_forceinline void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { +extern mi_decl_forceinline void* _mi_xtheap_malloc_zero(mi_theap_t* xtheap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { // fast path for small objects #if MI_THEAP_INITASNULL - if mi_likely(theap!=NULL && size <= MI_SMALL_SIZE_MAX) + if mi_likely(xtheap!=NULL && size <= MI_SMALL_SIZE_MAX) #else if mi_likely(size <= MI_SMALL_SIZE_MAX) #endif { mi_assert_internal(huge_alignment == 0); - return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, zero, ppage); + return mi_theap_xmalloc_small_zero(xtheap, size, false, zero, ppage); } else { - return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage); + return mi_xtheap_malloc_generic(xtheap, size, zero, huge_alignment, ppage); } } @@ -276,80 +288,92 @@ extern mi_decl_forceinline void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept { mi_assert(theap!=NULL); - return mi_theap_malloc_zero_nonnull(theap, size, false, 0, NULL); + return mi_theap_malloc_zero(theap, size, false, 0, NULL); +} + +static mi_decl_restrict void* mi_xtheap_malloc(mi_theap_t* xtheap, size_t size) mi_attr_noexcept { + return _mi_xtheap_malloc_zero(xtheap, size, false, 0, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept { - return _mi_theap_malloc_zero(_mi_theap_default(), size, false, 0, NULL); + return mi_xtheap_malloc(_mi_theap_default(), size); } mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, 0, NULL); + return mi_theap_malloc(_mi_heap_theap_get_or_init(heap), size); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { // return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, false, 0, NULL); - mi_theap_t* const theap = _mi_heap_theap_cached(heap); - if mi_likely(theap!=NULL) { return mi_theap_malloc_zero_nonnull(theap, size, false, 0, NULL); } - else { return mi_heap_init_malloc(heap, size); } + mi_theap_t* const xtheap = _mi_heap_theap_cached(heap); + if mi_likely(xtheap!=NULL) { return mi_theap_malloc(xtheap, size); } + else { return mi_heap_init_malloc(heap, size); } } // zero initialized small block mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept { - return mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, true, NULL); + return mi_xtheap_xmalloc_small_zero(_mi_theap_default(), size, false, true, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_wzalloc_small(size_t wsize) mi_attr_noexcept { - return mi_theap_xmalloc_small_zero(_mi_theap_default(), wsize, true, true, NULL); + return mi_xtheap_xmalloc_small_zero(_mi_theap_default(), wsize, true, true, NULL); } mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { mi_assert(theap!=NULL); - return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, true, NULL); + return mi_theap_xmalloc_small_zero(theap, size, false, true, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_theap_wzalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept { mi_assert(theap!=NULL); - return mi_theap_xmalloc_small_zero_nonnull(theap, wsize, true, true, NULL); + return mi_theap_xmalloc_small_zero(theap, wsize, true, true, NULL); } mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_xmalloc_small_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, true, NULL); + return mi_theap_zalloc_small(_mi_heap_theap_get_or_init(heap), size); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { // optimize: return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, true, NULL); - mi_theap_t* const theap = _mi_heap_theap_cached(heap); - if mi_likely(theap!=NULL) { return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, true, NULL); } - else { return mi_heap_init_zalloc_small(heap, size); } + mi_theap_t* const xtheap = _mi_heap_theap_cached(heap); + if mi_likely(xtheap!=NULL) { return mi_theap_zalloc_small(xtheap, size); } + else { return mi_heap_init_zalloc_small(heap, size); } } mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept { - return _mi_theap_malloc_zero(theap, size, true, 0, NULL); + mi_assert(theap!=NULL); + return mi_theap_malloc_zero(theap, size, true, 0, NULL); +} + +mi_decl_nodiscard static mi_decl_restrict void* mi_xtheap_zalloc(mi_theap_t* xtheap, size_t size) mi_attr_noexcept { + return _mi_xtheap_malloc_zero(xtheap, size, true, 0, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept { - return _mi_theap_malloc_zero(_mi_theap_default(), size, true, 0, NULL); + return _mi_xtheap_malloc_zero(_mi_theap_default(), size, true, 0, NULL); } mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, true, 0, NULL); + return mi_theap_zalloc(_mi_heap_theap_get_or_init(heap), size); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { // optimize: return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, true, 0, NULL); - mi_theap_t* const theap = _mi_heap_theap_cached(heap); - if mi_likely(theap!=NULL) { return mi_theap_malloc_zero_nonnull(theap, size, true, 0, NULL); } - else { return mi_heap_init_zalloc(heap, size); } + mi_theap_t* const xtheap = _mi_heap_theap_cached(heap); + if mi_likely(xtheap!=NULL) { return mi_theap_zalloc(xtheap, size); } + else { return mi_heap_init_zalloc(heap, size); } } mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept { + mi_assert(theap!=NULL); size_t total; if (mi_count_size_overflow(count,size,&total)) return NULL; return mi_theap_zalloc(theap,total); } mi_decl_nodiscard mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept { - return mi_theap_calloc(_mi_theap_default(),count,size); + size_t total; + if (mi_count_size_overflow(count,size,&total)) return NULL; + return mi_zalloc(total); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept { @@ -369,25 +393,25 @@ static void* mi_ublock_size( void* p, mi_page_t* page, size_t* pblock_size ) { // Return usable size mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, false, &page); + void* p = mi_xtheap_xmalloc_small_zero(_mi_theap_default(), size, false, false, &page); return mi_ublock_size(p,page,pblock_size); } mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, true, &page); + void* p = mi_xtheap_xmalloc_small_zero(_mi_theap_default(), size, false, true, &page); return mi_ublock_size(p,page,pblock_size); } mi_decl_nodiscard mi_decl_restrict void* mi_umalloc(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, false, 0, &page); + void* p = _mi_xtheap_malloc_zero(_mi_theap_default(), size, false, 0, &page); return mi_ublock_size(p,page,pblock_size); } mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, true, 0, &page); + void* p = _mi_xtheap_malloc_zero(_mi_theap_default(), size, true, 0, &page); return mi_ublock_size(p,page,pblock_size); } @@ -398,14 +422,10 @@ mi_decl_nodiscard mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, s } // Uninitialized `calloc` -static mi_decl_restrict void* mi_theap_mallocn(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept { size_t total; if (mi_count_size_overflow(count, size, &total)) return NULL; - return mi_theap_malloc(theap, total); -} - -mi_decl_nodiscard mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept { - return mi_theap_mallocn(_mi_theap_default(),count,size); + return mi_malloc(total); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept { @@ -430,7 +450,7 @@ void* mi_expand(void* p, size_t newsize) mi_attr_noexcept { #endif } -static mi_decl_forceinline void* mi_theap_realloc_zero_ex(mi_theap_t* theap, void* p, size_t newsize, bool zero, size_t* pblock_size_pre, size_t* pblock_size_post) mi_attr_noexcept { +static mi_decl_forceinline void* mi_xtheap_realloc_zero_ex(mi_theap_t* xtheap, void* p, size_t newsize, bool zero, size_t* pblock_size_pre, size_t* pblock_size_post) mi_attr_noexcept { // if p == NULL then behave as malloc. // else if size == 0 then reallocate to a zero-sized block (and don't return NULL, just as mi_malloc(0)). // (this means that returning NULL always indicates an error, and `p` will not have been freed in that case.) @@ -455,10 +475,10 @@ static mi_decl_forceinline void* mi_theap_realloc_zero_ex(mi_theap_t* theap, voi if mi_unlikely(newsize<=size && newsize>=(size/2) && newsize>0) { // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0) mi_assert_internal(page!=NULL); // note: page!=NULL (since if p==NULL, we have size=0 and size>=newsize>0 #if MI_THEAP_INITASNULL - if (theap!=NULL) + if (xtheap!=NULL) #endif { - if (mi_page_heap(page)==_mi_theap_heap_peek(theap)) { // and within the same heap + if (mi_page_heap(page)==_mi_theap_heap_peek(xtheap)) { // and within the same heap mi_assert_internal(p!=NULL); // todo: do not track as the usable size is still the same in the free; adjust potential padding? // mi_track_resize(p,size,newsize) @@ -470,7 +490,7 @@ static mi_decl_forceinline void* mi_theap_realloc_zero_ex(mi_theap_t* theap, voi } // note: we don't zero allocate upfront so we only zero initialize the expanded part mi_page_t* newpage; // use block_size for zero-ing, issue #763 - void* const newp = _mi_theap_malloc_zero(theap,newsize,false /* no zero */,0,&newpage); + void* const newp = _mi_xtheap_malloc_zero(xtheap,newsize,false /* no zero */,0,&newpage); if mi_likely(newp != NULL) { if (pblock_size_post!=NULL) { *pblock_size_post = mi_page_block_size(newpage); } const size_t copy_size = (newsize > size ? size : newsize); @@ -492,64 +512,56 @@ static mi_decl_forceinline void* mi_theap_realloc_zero_ex(mi_theap_t* theap, voi return newp; } -void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero) mi_attr_noexcept { - return mi_theap_realloc_zero_ex(theap,p,newsize,zero,NULL,NULL); +void* _mi_xtheap_realloc_zero(mi_theap_t* xtheap, void* p, size_t newsize, bool zero) mi_attr_noexcept { + return mi_xtheap_realloc_zero_ex(xtheap,p,newsize,zero,NULL,NULL); } -mi_decl_nodiscard void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept { - // optimize p==NULL +// for theap_realloc/rezalloc we allow theap==NULL to reduce variants +mi_decl_nodiscard void* mi_theap_realloc(mi_theap_t* xtheap, void* p, size_t newsize) mi_attr_noexcept { if (p==NULL) { - return mi_theap_malloc(theap,newsize); + return mi_xtheap_malloc(xtheap,newsize); } else { - return _mi_theap_realloc_zero(theap, p, newsize, false); + return _mi_xtheap_realloc_zero(xtheap, p, newsize, false); } } -static void* mi_theap_reallocn(mi_theap_t* theap, void* p, size_t count, size_t size) mi_attr_noexcept { - size_t total; - if (mi_count_size_overflow(count, size, &total)) return NULL; - return mi_theap_realloc(theap, p, total); -} - - -// Reallocate but free `p` on errors -static void* mi_theap_reallocf(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept { - void* newp = mi_theap_realloc(theap, p, newsize); - if (newp==NULL && p!=NULL) mi_free(p); - return newp; -} - -mi_decl_nodiscard void* mi_theap_rezalloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept { - // optimize p==NULL +mi_decl_nodiscard void* mi_theap_rezalloc(mi_theap_t* xtheap, void* p, size_t newsize) mi_attr_noexcept { if (p==NULL) { - return mi_theap_zalloc(theap,newsize); + return mi_xtheap_zalloc(xtheap,newsize); } else { - return _mi_theap_realloc_zero(theap, p, newsize, true); + return _mi_xtheap_realloc_zero(xtheap, p, newsize, true); } } -static void* mi_theap_recalloc(mi_theap_t* theap, void* p, size_t count, size_t size) mi_attr_noexcept { +static void* mi_theap_recalloc(mi_theap_t* xtheap, void* p, size_t count, size_t size) mi_attr_noexcept { size_t total; if (mi_count_size_overflow(count, size, &total)) return NULL; - return mi_theap_rezalloc(theap, p, total); + return mi_theap_rezalloc(xtheap, p, total); } - mi_decl_nodiscard void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept { return mi_theap_realloc(_mi_theap_default(),p,newsize); } mi_decl_nodiscard void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept { - return mi_theap_reallocn(_mi_theap_default(),p,count,size); + size_t total; + if (mi_count_size_overflow(count, size, &total)) return NULL; + return mi_theap_realloc(_mi_theap_default(),p,total); } mi_decl_nodiscard void* mi_urealloc(void* p, size_t newsize, size_t* pblock_size_pre, size_t* pblock_size_post) mi_attr_noexcept { - return mi_theap_realloc_zero_ex(_mi_theap_default(),p,newsize, false, pblock_size_pre, pblock_size_post); + return mi_xtheap_realloc_zero_ex(_mi_theap_default(),p,newsize, false, pblock_size_pre, pblock_size_post); } // Reallocate but free `p` on errors +static void* mi_theap_reallocf(mi_theap_t* xtheap, void* p, size_t newsize) mi_attr_noexcept { + void* newp = mi_theap_realloc(xtheap, p, newsize); + if (newp==NULL && p!=NULL) mi_free(p); + return newp; +} + mi_decl_nodiscard void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept { return mi_theap_reallocf(_mi_theap_default(),p,newsize); } @@ -568,7 +580,9 @@ mi_decl_nodiscard void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize } mi_decl_nodiscard void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept { - return mi_theap_reallocn(_mi_heap_theap(heap), p, count, size); + size_t total; + if (mi_count_size_overflow(count, size, &total)) return NULL; + return mi_theap_realloc(_mi_heap_theap(heap), p, total); } // Reallocate but free `p` on errors @@ -591,11 +605,11 @@ mi_decl_nodiscard void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t count, // ------------------------------------------------------ // `strdup` using mi_malloc -mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strdup(mi_theap_t* theap, const char* s) mi_attr_noexcept { +mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strdup(mi_theap_t* xtheap, const char* s) mi_attr_noexcept { if (s == NULL) return NULL; size_t len = _mi_strlen(s); if (len > MI_MAX_ALLOC_SIZE - 1) return NULL; // prevent overflow on len+1 - char* t = (char*)mi_theap_malloc(theap,len+1); + char* t = (char*)mi_xtheap_malloc(xtheap,len+1); if (t == NULL) return NULL; _mi_memcpy(t, s, len); t[len] = 0; @@ -611,11 +625,11 @@ mi_decl_nodiscard mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const c } // `strndup` using mi_malloc -mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strndup(mi_theap_t* theap, const char* s, size_t n) mi_attr_noexcept { +mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strndup(mi_theap_t* xtheap, const char* s, size_t n) mi_attr_noexcept { if (s == NULL) return NULL; const size_t len = _mi_strnlen(s,n); // len <= n if (len > MI_MAX_ALLOC_SIZE - 1) return NULL; // prevent overflow on len+1 - char* t = (char*)mi_theap_malloc(theap, len+1); + char* t = (char*)mi_xtheap_malloc(xtheap, len+1); if (t == NULL) return NULL; _mi_memcpy(t, s, len); t[len] = 0; @@ -637,7 +651,7 @@ mi_decl_nodiscard mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const #define PATH_MAX MAX_PATH #endif -mi_decl_nodiscard static mi_decl_restrict char* mi_theap_realpath(mi_theap_t* theap, const char* fname, char* resolved_name) mi_attr_noexcept { +mi_decl_nodiscard static mi_decl_restrict char* mi_theap_realpath(mi_theap_t* xtheap, const char* fname, char* resolved_name) mi_attr_noexcept { // todo: use GetFullPathNameW to allow longer file names if (fname==NULL || *fname==0) { errno = EINVAL; @@ -668,7 +682,7 @@ mi_decl_nodiscard static mi_decl_restrict char* mi_theap_realpath(mi_theap_t* th return resolved_name; } else { - return mi_theap_strndup(theap, buf, PATH_MAX); + return mi_theap_strndup(xtheap, buf, PATH_MAX); } } @@ -694,7 +708,7 @@ static size_t mi_path_max(void) { return pmax; } -char* mi_theap_realpath(mi_theap_t* theap, const char* fname, char* resolved_name) mi_attr_noexcept { +char* mi_theap_realpath(mi_theap_t* xtheap, const char* fname, char* resolved_name) mi_attr_noexcept { if (resolved_name != NULL) { return realpath(fname,resolved_name); } @@ -708,13 +722,13 @@ char* mi_theap_realpath(mi_theap_t* theap, const char* fname, char* resolved_nam return result; */ const size_t n = mi_path_max(); - char* const buf = (char*)mi_zalloc(n+1); + char* const buf = (char*)mi_xtheap_zalloc(xtheap,n+1); if (buf == NULL) { errno = ENOMEM; return NULL; } char* rname = realpath(fname,buf); - char* result = mi_theap_strndup(theap,rname,n); // ok if `rname==NULL` + char* result = mi_theap_strndup(xtheap,rname,n); // ok if `rname==NULL` mi_free(buf); return result; } @@ -809,11 +823,11 @@ static bool mi_try_new_handler(bool nothrow) { } #endif -static mi_decl_noinline void* mi_theap_try_new(mi_theap_t* theap, size_t size, bool nothrow ) { +static mi_decl_noinline void* mi_xtheap_try_new(mi_theap_t* xtheap, size_t size, bool nothrow ) { void* p = NULL; for(int i = 0; i < MI_TRY_NEW_MAX && p == NULL && mi_try_new_handler(nothrow); i++) { if (size > MI_MAX_ALLOC_SIZE) return NULL; // call try_new_handler at least once - p = mi_theap_malloc(theap,size); + p = mi_xtheap_malloc(xtheap,size); } return p; } @@ -823,12 +837,13 @@ static mi_decl_noinline void* mi_theap_try_new(mi_theap_t* theap, size_t size, b // } static mi_decl_noinline void* mi_heap_try_new(mi_heap_t* heap, size_t size, bool nothrow) { - return mi_theap_try_new(_mi_heap_theap(heap), size, nothrow); + return mi_xtheap_try_new(_mi_heap_theap(heap), size, nothrow); } -mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* theap, size_t size) { - void* p = mi_theap_malloc(theap,size); - if mi_unlikely(p == NULL) return mi_theap_try_new(theap, size, false); +// again, to reduce variants we allow `xtheap==NULL`. +mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* xtheap, size_t size) { + void* p = mi_xtheap_malloc(xtheap,size); + if mi_unlikely(p == NULL) return mi_xtheap_try_new(xtheap, size, false); return p; } @@ -842,14 +857,14 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size return p; } -mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* theap, size_t count, size_t size) { +mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* xtheap, size_t count, size_t size) { size_t total; if mi_unlikely(mi_count_size_overflow(count, size, &total)) { mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc return NULL; } else { - return mi_theap_alloc_new(theap,total); + return mi_theap_alloc_new(xtheap,total); } } @@ -862,9 +877,9 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, si } -mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new_nothrow(mi_theap_t* theap, size_t size) mi_attr_noexcept { - void* p = mi_theap_malloc(theap,size); - if mi_unlikely(p == NULL) return mi_theap_try_new(theap, size, true); +mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new_nothrow(mi_theap_t* xtheap, size_t size) mi_attr_noexcept { + void* p = mi_xtheap_malloc(xtheap,size); + if mi_unlikely(p == NULL) return mi_xtheap_try_new(xtheap, size, true); return p; } @@ -926,7 +941,7 @@ mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) { #ifdef __cplusplus void* _mi_externs[] = { (void*)&_mi_page_malloc_zero, - (void*)&_mi_theap_malloc_zero, + (void*)&_mi_xtheap_malloc_zero, (void*)&mi_theap_malloc, (void*)&mi_theap_zalloc, (void*)&mi_theap_malloc_small, diff --git a/test/test-api.c b/test/test-api.c index 1f6255926..de52d642d 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -55,6 +55,9 @@ bool test_stl_theap_allocator3(void); bool test_stl_theap_allocator4(void); static bool test_zero_aligned_first(void); +#ifdef __cplusplus +static bool test_new_first(void); +#endif static bool mem_has_vals(const uint8_t* p, size_t size, uint8_t val) { if (p==NULL) return false; @@ -569,10 +572,16 @@ int main(void) { // --------------------------------------------------- // Threads // --------------------------------------------------- - CHECK_BODY("zero_aligned_first") { + CHECK_BODY("thread_zero_aligned_first") { result = mi_run_on_thread(&test_zero_aligned_first); } + #ifdef __cplusplus + CHECK_BODY("thread_new_first") { + result = mi_run_on_thread(&test_new_first); + } + #endif + //mi_stats_print(NULL); // --------------------------------------------------- @@ -750,3 +759,12 @@ static bool test_zero_aligned_first(void) { } +#ifdef __cplusplus +static bool test_new_first(void) { + char* p = (char*)mi_new(20); + bool res = (p != NULL && (uintptr_t)(p) % 16 == 0); + mi_free_csize(p,20); + return res; +} +#endif + From a09da20280901a1ff7f53bc151fb3e6a30b7a07e Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 14 Sep 2026 22:57:33 -0700 Subject: [PATCH 07/24] fix recursion --- src/alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/alloc.c b/src/alloc.c index e81bfa36a..9b8f5d97e 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -218,7 +218,7 @@ static mi_decl_forceinline mi_decl_restrict void* mi_xtheap_xmalloc_small_zero(m return mi_xtheap_malloc_generic(xtheap, (is_wsize ? xsize * MI_SIZE_SIZE : xsize), zero, 0, ppage); // tailcall } #else - return mi_xtheap_xmalloc_small_zero(xtheap, xsize, is_wsize, zero, ppage); + return mi_theap_xmalloc_small_zero(xtheap, xsize, is_wsize, zero, ppage); #endif } From e2f06aabf7d3cbb2e59841baa9ef67b64d8f013d Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 14 Sep 2026 23:07:00 -0700 Subject: [PATCH 08/24] weaken assertion --- src/free.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/free.c b/src/free.c index 223831dd4..37bb71842 100644 --- a/src/free.c +++ b/src/free.c @@ -248,7 +248,7 @@ static mi_decl_forceinline bool mi_ptr_page_is_valid_ex(const void* p, const cha mi_assert_internal(page!=NULL); mi_assert(cpage==page /* page_map lookup should be the same as aligned lookup */ ); #if !MI_GUARDED - if (free_small) { mi_assert_internal(page->block_size <= mi_good_size(MI_SMALL_SIZE_MAX) /* free small should only be called on small pages */); } + if (free_small) { mi_assert_internal(page->block_size <= MI_SMALL_MAX_OBJ_SIZE) /* free small should only be called on small pages */); } #endif *ppage = page; return true; From d8913a5986bb3f6d2409f50558e6d9723e41b011 Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 14 Sep 2026 23:09:58 -0700 Subject: [PATCH 09/24] fix syntax --- src/free.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/free.c b/src/free.c index 37bb71842..dc6c5fa0e 100644 --- a/src/free.c +++ b/src/free.c @@ -248,7 +248,7 @@ static mi_decl_forceinline bool mi_ptr_page_is_valid_ex(const void* p, const cha mi_assert_internal(page!=NULL); mi_assert(cpage==page /* page_map lookup should be the same as aligned lookup */ ); #if !MI_GUARDED - if (free_small) { mi_assert_internal(page->block_size <= MI_SMALL_MAX_OBJ_SIZE) /* free small should only be called on small pages */); } + if (free_small) { mi_assert_internal(page->block_size <= MI_SMALL_MAX_OBJ_SIZE); /* free small should only be called on small pages */ } #endif *ppage = page; return true; From d1e9bf45cb2f4f8f59232fcaf2e6f71b4cc51eee Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 14 Sep 2026 23:24:34 -0700 Subject: [PATCH 10/24] temporarily only reclaim when the counter is 1 --- src/free.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/free.c b/src/free.c index dc6c5fa0e..812e48121 100644 --- a/src/free.c +++ b/src/free.c @@ -84,7 +84,7 @@ static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was do { mi_block_set_next(page, block, mi_tf_block(tf_old)); const size_t counter = mi_tf_counter(tf_old); - const bool try_reclaim = (allow_reclaim && (counter==1 || theap==page->theap)) || // always try to reclaim in our own heap + const bool try_reclaim = (allow_reclaim && (counter==1 /*|| theap==page->theap*/)) || // always try to reclaim in our own heap (counter==1 && !mi_tf_is_owned(tf_old)); // must try to reclaim if this is (possibly) the last block in an unowned page so we can free it const bool new_owned = (try_reclaim ? true : mi_tf_is_owned(tf_old)); // if allow collection then always try to claim it if the page is abandoned tf_new = mi_tf_create(block, new_owned, (counter<=1 ? counter : counter - 1)); From fe94a2d9678b580e6b72f73d555969795da82abe Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 14 Sep 2026 23:25:00 -0700 Subject: [PATCH 11/24] temporarily only reclaim when the counter is 1 --- src/free.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/free.c b/src/free.c index 812e48121..eac2be243 100644 --- a/src/free.c +++ b/src/free.c @@ -78,7 +78,7 @@ static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was #endif // push atomically on the page thread free list - mi_theap_t* theap = _mi_page_associated_theap_peek(page); + // mi_theap_t* theap = _mi_page_associated_theap_peek(page); mi_thread_free_t tf_new; mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free); do { From d184163c4588f035540cd3099cf138a2fc1cc1c6 Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 14 Sep 2026 23:36:23 -0700 Subject: [PATCH 12/24] make page->theap atomic so we can read it unowned --- include/mimalloc/internal.h | 23 ++++++++++++----------- include/mimalloc/types.h | 2 +- src/free.c | 6 +++--- src/init.c | 2 +- src/page-queue.c | 2 +- src/page.c | 9 ++++----- src/subproc.c | 2 +- 7 files changed, 23 insertions(+), 23 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 44aa92f20..9871b3e4a 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1025,6 +1025,14 @@ static inline size_t mi_page_min_commit_size(void) { // Page thread id and flags //----------------------------------------------------------- +// Get the theap associated with this page. +static inline mi_theap_t* mi_page_theap(const mi_page_t* page) { + // mi_assert_internal(!mi_page_is_abandoned(page)); + mi_theap_t* theap = mi_atomic_load_ptr_relaxed(mi_theap_t,&page->xtheap); + mi_assert_internal(mi_theap_is_initialized(theap)); + return theap; +} + // Thread id of thread that owns this page (with flags in the bottom 2 bits) static inline mi_threadid_t mi_page_xthread_id(const mi_page_t* page) { return mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_id); @@ -1054,7 +1062,7 @@ static inline void mi_page_set_in_full(mi_page_t* page, bool in_full) { const bool was_in_full = mi_page_flags_set(page, in_full, MI_PAGE_IN_FULL_QUEUE); if (was_in_full != in_full) { // optimize: maintain pages_full_size to avoid visiting the full queue (issue #1220) - mi_theap_t* const theap = page->theap; + mi_theap_t* const theap = mi_page_theap(page); mi_assert_internal(theap!=NULL); if (theap != NULL) { mi_assert_internal(page->capacity==page->reserved); @@ -1075,7 +1083,7 @@ static inline void mi_page_set_has_interior_pointers(mi_page_t* page, bool has_a static inline void mi_page_set_theap(mi_page_t* page, mi_theap_t* theap) { // mi_assert_internal(!mi_page_is_in_full(page)); // can happen when destroying pages on theap_destroy - page->theap = theap; + mi_atomic_store_ptr_relaxed(mi_theap_t,&page->xtheap, theap); const mi_threadid_t tid = (theap == NULL ? MI_THREADID_ABANDONED : theap->tld->thread_id); mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0); @@ -1107,19 +1115,12 @@ static inline void mi_page_clear_abandoned_mapped(mi_page_t* page) { } -static inline mi_theap_t* mi_page_theap(const mi_page_t* page) { - mi_assert_internal(!mi_page_is_abandoned(page)); - mi_assert_internal(page->theap != NULL && page->theap != &_mi_theap_empty); - return page->theap; -} - static inline mi_tld_t* mi_page_tld(const mi_page_t* page) { mi_assert_internal(!mi_page_is_abandoned(page)); - mi_assert_internal(page->theap != NULL); - return page->theap->tld; + mi_assert_internal(mi_page_theap(page) != NULL); + return mi_page_theap(page)->tld; } - static inline mi_heap_t* mi_page_heap(const mi_page_t* page) { mi_heap_t* heap = page->heap; mi_assert_internal(heap != NULL); diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 530c464a3..d0ffc7714 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -487,7 +487,7 @@ typedef struct mi_page_s { // next cache line _Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads (= `mi_block_t* | (1 if owned)`) - mi_theap_t* theap; // the theap owning this page (may not be valid or NULL for abandoned pages) + _Atomic(mi_theap_t*) xtheap; // the theap owning this page (may not be valid or NULL for abandoned pages) mi_heap_t* heap; // const: the heap owning this page struct mi_page_s* next; // next page owned by the theap with the same `block_size` diff --git a/src/free.c b/src/free.c index eac2be243..d80e46223 100644 --- a/src/free.c +++ b/src/free.c @@ -78,13 +78,13 @@ static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was #endif // push atomically on the page thread free list - // mi_theap_t* theap = _mi_page_associated_theap_peek(page); + mi_theap_t* theap = _mi_page_associated_theap_peek(page); mi_thread_free_t tf_new; mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free); do { mi_block_set_next(page, block, mi_tf_block(tf_old)); const size_t counter = mi_tf_counter(tf_old); - const bool try_reclaim = (allow_reclaim && (counter==1 /*|| theap==page->theap*/)) || // always try to reclaim in our own heap + const bool try_reclaim = (allow_reclaim && (counter==1 || theap==mi_page_theap(page))) || // always try to reclaim in our own heap (counter==1 && !mi_tf_is_owned(tf_old)); // must try to reclaim if this is (possibly) the last block in an unowned page so we can free it const bool new_owned = (try_reclaim ? true : mi_tf_is_owned(tf_old)); // if allow collection then always try to claim it if the page is abandoned tf_new = mi_tf_create(block, new_owned, (counter<=1 ? counter : counter - 1)); @@ -497,7 +497,7 @@ static mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, long // todo: cache `is_in_threadpool` and `exclusive_arena` directly in the theap for performance? // set max_reclaim limit long max_reclaim = 0; - if mi_likely(theap == page->theap) { // did this page originate from the current theap? (and thus allocated from this thread) + if mi_likely(theap == mi_page_theap(page)) { // did this page originate from the current theap? (and thus allocated from this thread) // originating theap max_reclaim = _mi_option_get_fast(theap->tld->is_in_threadpool ? mi_option_page_cross_thread_max_reclaim : mi_option_page_max_reclaim); } diff --git a/src/init.c b/src/init.c index 9b0ded5b5..30633228e 100644 --- a/src/init.c +++ b/src/init.c @@ -33,7 +33,7 @@ static const mi_page_t mi_page_empty = { 0, // retire_expire false, // is_zero MI_ATOMIC_VAR_INIT(0), // xthread_free - NULL, // theap + MI_ATOMIC_VAR_INIT(NULL), // xtheap NULL, // heap NULL, NULL, // next, prev MI_MEMID_STATIC, // memid diff --git a/src/page-queue.c b/src/page-queue.c index c8e9b5425..9285b1597 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -160,7 +160,7 @@ bool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq) { else { mi_assert_internal(mi_page_block_size(page) == pq->block_size); } - mi_assert_internal(page->theap == theap); + mi_assert_internal(mi_page_theap(page) == theap); if (page->next == NULL) { mi_assert_internal(pq->last == page); } diff --git a/src/page.c b/src/page.c index f26657788..aa21f747e 100644 --- a/src/page.c +++ b/src/page.c @@ -148,7 +148,7 @@ bool _mi_page_is_valid(mi_page_t* page) { #if MI_STATS || MI_SAMPLE // Gets the theap belonging to a page. static mi_theap_t* mi_theap_of_page(mi_page_t* page) { - mi_theap_t* theap = page->theap; + mi_theap_t* theap = mi_page_theap(page); if mi_unlikely(mi_page_thread_id(page) != _mi_prim_thread_id()) { theap = _mi_page_associated_theap_peek(page); } @@ -475,9 +475,9 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) { } else { mi_page_queue_remove(pq, page); - mi_theap_t* theap = page->theap; + mi_theap_t* theap = mi_page_theap(page); mi_page_set_theap(page, NULL); - page->theap = theap; // don't actually set theap to NULL so we can reclaim_on_free within the same theap + mi_atomic_store_ptr_relaxed(mi_theap_t, &page->xtheap, theap);// don't actually set theap to NULL so we can reclaim_on_free within the same theap _mi_arenas_page_abandon(page, theap); // _mi_arenas_collect(false, false, theap->tld); // allow purging } @@ -909,8 +909,7 @@ mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) { mi_assert_internal(page->heap != NULL); mi_assert_internal(page->heap == _mi_theap_heap(theap)); - mi_assert_internal(page->theap!=NULL); - mi_assert_internal(page->theap == mi_page_theap(page)); + mi_assert_internal(mi_page_theap(page)!=NULL); mi_assert_internal(page->capacity == 0); mi_assert_internal(page->free == NULL); mi_assert_internal(mi_page_used(page) == 0); diff --git a/src/subproc.c b/src/subproc.c index f42d4a11c..0e48ff6cf 100644 --- a/src/subproc.c +++ b/src/subproc.c @@ -83,7 +83,7 @@ void _mi_meta_free(mi_subproc_t* subproc, void* p, mi_memid_t memid) { bool _mi_meta_is_meta_page(const mi_subproc_t* subproc, const mi_page_t* page) { if (page==NULL) return false; - mi_theap_t* theap = page->theap; + mi_theap_t* theap = mi_page_theap(page); return (theap != NULL && theap == subproc->theap_meta); } From b1034d6ebb7d5544a78e77d13b00b110e62848b8 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 15 Sep 2026 09:51:29 -0700 Subject: [PATCH 13/24] Revert "make page->theap atomic so we can read it unowned" This reverts commit d184163c4588f035540cd3099cf138a2fc1cc1c6. --- include/mimalloc/internal.h | 23 +++++++++++------------ include/mimalloc/types.h | 2 +- src/free.c | 6 +++--- src/init.c | 2 +- src/page-queue.c | 2 +- src/page.c | 9 +++++---- src/subproc.c | 2 +- 7 files changed, 23 insertions(+), 23 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 9871b3e4a..44aa92f20 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1025,14 +1025,6 @@ static inline size_t mi_page_min_commit_size(void) { // Page thread id and flags //----------------------------------------------------------- -// Get the theap associated with this page. -static inline mi_theap_t* mi_page_theap(const mi_page_t* page) { - // mi_assert_internal(!mi_page_is_abandoned(page)); - mi_theap_t* theap = mi_atomic_load_ptr_relaxed(mi_theap_t,&page->xtheap); - mi_assert_internal(mi_theap_is_initialized(theap)); - return theap; -} - // Thread id of thread that owns this page (with flags in the bottom 2 bits) static inline mi_threadid_t mi_page_xthread_id(const mi_page_t* page) { return mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_id); @@ -1062,7 +1054,7 @@ static inline void mi_page_set_in_full(mi_page_t* page, bool in_full) { const bool was_in_full = mi_page_flags_set(page, in_full, MI_PAGE_IN_FULL_QUEUE); if (was_in_full != in_full) { // optimize: maintain pages_full_size to avoid visiting the full queue (issue #1220) - mi_theap_t* const theap = mi_page_theap(page); + mi_theap_t* const theap = page->theap; mi_assert_internal(theap!=NULL); if (theap != NULL) { mi_assert_internal(page->capacity==page->reserved); @@ -1083,7 +1075,7 @@ static inline void mi_page_set_has_interior_pointers(mi_page_t* page, bool has_a static inline void mi_page_set_theap(mi_page_t* page, mi_theap_t* theap) { // mi_assert_internal(!mi_page_is_in_full(page)); // can happen when destroying pages on theap_destroy - mi_atomic_store_ptr_relaxed(mi_theap_t,&page->xtheap, theap); + page->theap = theap; const mi_threadid_t tid = (theap == NULL ? MI_THREADID_ABANDONED : theap->tld->thread_id); mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0); @@ -1115,12 +1107,19 @@ static inline void mi_page_clear_abandoned_mapped(mi_page_t* page) { } +static inline mi_theap_t* mi_page_theap(const mi_page_t* page) { + mi_assert_internal(!mi_page_is_abandoned(page)); + mi_assert_internal(page->theap != NULL && page->theap != &_mi_theap_empty); + return page->theap; +} + static inline mi_tld_t* mi_page_tld(const mi_page_t* page) { mi_assert_internal(!mi_page_is_abandoned(page)); - mi_assert_internal(mi_page_theap(page) != NULL); - return mi_page_theap(page)->tld; + mi_assert_internal(page->theap != NULL); + return page->theap->tld; } + static inline mi_heap_t* mi_page_heap(const mi_page_t* page) { mi_heap_t* heap = page->heap; mi_assert_internal(heap != NULL); diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index d0ffc7714..530c464a3 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -487,7 +487,7 @@ typedef struct mi_page_s { // next cache line _Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads (= `mi_block_t* | (1 if owned)`) - _Atomic(mi_theap_t*) xtheap; // the theap owning this page (may not be valid or NULL for abandoned pages) + mi_theap_t* theap; // the theap owning this page (may not be valid or NULL for abandoned pages) mi_heap_t* heap; // const: the heap owning this page struct mi_page_s* next; // next page owned by the theap with the same `block_size` diff --git a/src/free.c b/src/free.c index d80e46223..eac2be243 100644 --- a/src/free.c +++ b/src/free.c @@ -78,13 +78,13 @@ static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was #endif // push atomically on the page thread free list - mi_theap_t* theap = _mi_page_associated_theap_peek(page); + // mi_theap_t* theap = _mi_page_associated_theap_peek(page); mi_thread_free_t tf_new; mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free); do { mi_block_set_next(page, block, mi_tf_block(tf_old)); const size_t counter = mi_tf_counter(tf_old); - const bool try_reclaim = (allow_reclaim && (counter==1 || theap==mi_page_theap(page))) || // always try to reclaim in our own heap + const bool try_reclaim = (allow_reclaim && (counter==1 /*|| theap==page->theap*/)) || // always try to reclaim in our own heap (counter==1 && !mi_tf_is_owned(tf_old)); // must try to reclaim if this is (possibly) the last block in an unowned page so we can free it const bool new_owned = (try_reclaim ? true : mi_tf_is_owned(tf_old)); // if allow collection then always try to claim it if the page is abandoned tf_new = mi_tf_create(block, new_owned, (counter<=1 ? counter : counter - 1)); @@ -497,7 +497,7 @@ static mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, long // todo: cache `is_in_threadpool` and `exclusive_arena` directly in the theap for performance? // set max_reclaim limit long max_reclaim = 0; - if mi_likely(theap == mi_page_theap(page)) { // did this page originate from the current theap? (and thus allocated from this thread) + if mi_likely(theap == page->theap) { // did this page originate from the current theap? (and thus allocated from this thread) // originating theap max_reclaim = _mi_option_get_fast(theap->tld->is_in_threadpool ? mi_option_page_cross_thread_max_reclaim : mi_option_page_max_reclaim); } diff --git a/src/init.c b/src/init.c index 30633228e..9b0ded5b5 100644 --- a/src/init.c +++ b/src/init.c @@ -33,7 +33,7 @@ static const mi_page_t mi_page_empty = { 0, // retire_expire false, // is_zero MI_ATOMIC_VAR_INIT(0), // xthread_free - MI_ATOMIC_VAR_INIT(NULL), // xtheap + NULL, // theap NULL, // heap NULL, NULL, // next, prev MI_MEMID_STATIC, // memid diff --git a/src/page-queue.c b/src/page-queue.c index 9285b1597..c8e9b5425 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -160,7 +160,7 @@ bool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq) { else { mi_assert_internal(mi_page_block_size(page) == pq->block_size); } - mi_assert_internal(mi_page_theap(page) == theap); + mi_assert_internal(page->theap == theap); if (page->next == NULL) { mi_assert_internal(pq->last == page); } diff --git a/src/page.c b/src/page.c index aa21f747e..f26657788 100644 --- a/src/page.c +++ b/src/page.c @@ -148,7 +148,7 @@ bool _mi_page_is_valid(mi_page_t* page) { #if MI_STATS || MI_SAMPLE // Gets the theap belonging to a page. static mi_theap_t* mi_theap_of_page(mi_page_t* page) { - mi_theap_t* theap = mi_page_theap(page); + mi_theap_t* theap = page->theap; if mi_unlikely(mi_page_thread_id(page) != _mi_prim_thread_id()) { theap = _mi_page_associated_theap_peek(page); } @@ -475,9 +475,9 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) { } else { mi_page_queue_remove(pq, page); - mi_theap_t* theap = mi_page_theap(page); + mi_theap_t* theap = page->theap; mi_page_set_theap(page, NULL); - mi_atomic_store_ptr_relaxed(mi_theap_t, &page->xtheap, theap);// don't actually set theap to NULL so we can reclaim_on_free within the same theap + page->theap = theap; // don't actually set theap to NULL so we can reclaim_on_free within the same theap _mi_arenas_page_abandon(page, theap); // _mi_arenas_collect(false, false, theap->tld); // allow purging } @@ -909,7 +909,8 @@ mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) { mi_assert_internal(page->heap != NULL); mi_assert_internal(page->heap == _mi_theap_heap(theap)); - mi_assert_internal(mi_page_theap(page)!=NULL); + mi_assert_internal(page->theap!=NULL); + mi_assert_internal(page->theap == mi_page_theap(page)); mi_assert_internal(page->capacity == 0); mi_assert_internal(page->free == NULL); mi_assert_internal(mi_page_used(page) == 0); diff --git a/src/subproc.c b/src/subproc.c index 0e48ff6cf..f42d4a11c 100644 --- a/src/subproc.c +++ b/src/subproc.c @@ -83,7 +83,7 @@ void _mi_meta_free(mi_subproc_t* subproc, void* p, mi_memid_t memid) { bool _mi_meta_is_meta_page(const mi_subproc_t* subproc, const mi_page_t* page) { if (page==NULL) return false; - mi_theap_t* theap = mi_page_theap(page); + mi_theap_t* theap = page->theap; return (theap != NULL && theap == subproc->theap_meta); } From 8b3a5c324ca435be7b8fd364906805ec68e23b29 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 15 Sep 2026 16:33:06 -0700 Subject: [PATCH 14/24] revert to using a plain xthread_free without a counter --- include/mimalloc/internal.h | 40 +++++++------------------------------ src/arena.c | 2 +- src/free.c | 39 +++++++++++++++--------------------- src/page.c | 2 +- 4 files changed, 25 insertions(+), 58 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 44aa92f20..38973f185 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1155,46 +1155,20 @@ static inline bool _mi_is_process_heap_main(const mi_heap_t* heap) { // Thread free flag helpers static inline mi_block_t* mi_tf_block(mi_thread_free_t tf) { - #if MI_INTPTR_BITS - MI_MAX_VABITS >= 16 - return (mi_block_t*)(((tf & ~1) << 16) >> 16); - #else - return (mi_block_t*)(tf & ~1); - #endif + return (mi_block_t*)(tf & ~1); } + static inline bool mi_tf_is_owned(mi_thread_free_t tf) { return ((tf & 1) == 1); } -static inline size_t mi_tf_counter(mi_thread_free_t tf) { - #if MI_INTPTR_BITS - MI_MAX_VABITS >= 16 - return (size_t)((uintptr_t)tf >> (MI_INTPTR_BITS - 16)); - #else - return 1; - #endif -} -static inline mi_thread_free_t mi_tf_create(mi_block_t* block, bool owned, size_t counter) { - uintptr_t base = (uintptr_t)block | (owned ? 1 : 0); - #if MI_INTPTR_BITS - MI_MAX_VABITS >= 16 - mi_assert_internal(((base << 16) >> 16) == base); - base |= (uintptr_t)(counter) << (MI_INTPTR_BITS - 16); - #else - MI_UNUSED(counter); - #endif + +static inline mi_thread_free_t mi_tf_create(mi_block_t* block, bool owned) { + const uintptr_t base = (uintptr_t)block | (owned ? 1 : 0); return (mi_thread_free_t)base; } -// Create a new thread-free entry for the given page and block, with an appropriate counter based on the page's usage. -static inline mi_thread_free_t mi_page_tf_create(mi_page_t* page, mi_block_t* new_thread_free, bool owned) { - if (owned) { - return mi_tf_create(new_thread_free, owned, 0); - } - else { - size_t counter = mi_page_used(page); // after this many mt free's we should free the page - if (mi_page_is_full(page)) { - const uint16_t frac18 = 7 * (page->reserved / 8U); - if (frac18 < counter) { counter = (frac18 > 0 ? frac18 : 1); } // after this many mt free's we should reabandon to mapped - } - return mi_tf_create(new_thread_free, owned, counter); - } +static inline mi_thread_free_t mi_tf_set_owned(mi_thread_free_t tf, bool owned) { + return mi_tf_create(mi_tf_block(tf), owned); } // Thread free access diff --git a/src/arena.c b/src/arena.c index 7e45ab8c8..98fb5a084 100644 --- a/src/arena.c +++ b/src/arena.c @@ -646,7 +646,7 @@ static bool mi_abandoned_page_unown(mi_page_t* page, mi_theap_t* current_theapx) tf_old = mi_atomic_load_relaxed(&page->xthread_free); } mi_assert_internal(mi_tf_block(tf_old)==NULL); - tf_new = mi_page_tf_create(page, NULL, false); + tf_new = mi_tf_create(NULL, false); } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new)); return false; } diff --git a/src/free.c b/src/free.c index eac2be243..58899ecba 100644 --- a/src/free.c +++ b/src/free.c @@ -57,7 +57,7 @@ static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool } // Forward declaration for multi-threaded collect -static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free, bool allow_reclaim, size_t counter) mi_attr_noexcept; +static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free, bool allow_reclaim) mi_attr_noexcept; // Free a block multi-threaded static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded, bool allow_reclaim) mi_attr_noexcept @@ -83,23 +83,16 @@ static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free); do { mi_block_set_next(page, block, mi_tf_block(tf_old)); - const size_t counter = mi_tf_counter(tf_old); - const bool try_reclaim = (allow_reclaim && (counter==1 /*|| theap==page->theap*/)) || // always try to reclaim in our own heap - (counter==1 && !mi_tf_is_owned(tf_old)); // must try to reclaim if this is (possibly) the last block in an unowned page so we can free it - const bool new_owned = (try_reclaim ? true : mi_tf_is_owned(tf_old)); // if allow collection then always try to claim it if the page is abandoned - tf_new = mi_tf_create(block, new_owned, (counter<=1 ? counter : counter - 1)); + tf_new = mi_tf_create(block, true); } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new)); // and atomically try to collect the page if it was abandoned - if (allow_reclaim) { - const bool is_newly_owned = mi_tf_is_owned(tf_new) && !mi_tf_is_owned(tf_old); - if (is_newly_owned) { - mi_assert_internal(mi_page_is_abandoned(page)); - // mi_assert_internal(!mi_page_is_abandoned_mapped(page)); - mi_free_try_collect_mt(page, block, allow_reclaim, mi_tf_counter(tf_old)); - } + const bool is_newly_owned = mi_tf_is_owned(tf_new) && !mi_tf_is_owned(tf_old); + if (is_newly_owned) { + mi_assert_internal(mi_page_is_abandoned(page)); + // mi_assert_internal(!mi_page_is_abandoned_mapped(page)); + mi_free_try_collect_mt(page, block, allow_reclaim); } - } @@ -439,14 +432,14 @@ static bool mi_abandoned_page_try_reabandon_to_mapped(mi_page_t* page) // Release ownership of a page. This may free or reabandoned the page if other blocks are concurrently // freed in the meantime. Returns `true` if the page was freed. // By passing the captured `expected_thread_free`, we can often avoid calling `mi_page_free_collect`. -static void mi_abandoned_page_unown_from_free(mi_page_t* page, mi_block_t* expected_thread_free, size_t old_counter) { +static void mi_abandoned_page_unown_from_free(mi_page_t* page, mi_block_t* page_thread_free) { mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(!mi_page_all_free(page)); // try to cas atomically the original free list (`mt_free`) back with the ownership cleared. - mi_thread_free_t tf_expect = mi_tf_create(expected_thread_free, true, old_counter-1); - mi_thread_free_t tf_new = mi_page_tf_create(page, expected_thread_free, false); - while mi_unlikely(!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_expect, tf_new)) { + mi_thread_free_t tf_expect = mi_tf_create(page_thread_free,true); + mi_thread_free_t tf_new = mi_tf_set_owned(tf_expect,false); + while mi_unlikely(!mi_atomic_cas_strong_acq_rel(&page->xthread_free, &tf_expect, tf_new)) { mi_assert_internal(mi_tf_is_owned(tf_expect)); // while the xthread_free list is not empty.. while (mi_tf_block(tf_expect) != NULL) { @@ -455,12 +448,12 @@ static void mi_abandoned_page_unown_from_free(mi_page_t* page, mi_block_t* expec if (mi_abandoned_page_try_free(page)) return; if (mi_abandoned_page_try_reabandon_to_mapped(page)) return; // otherwise continue un-owning - tf_expect = mi_atomic_load_relaxed(&page->xthread_free); + tf_expect = mi_atomic_load_acquire(&page->xthread_free); } // and try again to release ownership - // mi_subproc_stat_increase(mi_page_subproc(page), pages_unabandon_busy_wait, 1); + mi_subproc_stat_counter_increase(mi_page_subproc(page), pages_unabandon_busy_wait, 1); mi_assert_internal(mi_tf_block(tf_expect)==NULL); - tf_new = mi_page_tf_create(page, NULL, false); + tf_new = mi_tf_create(NULL, false); } } @@ -524,7 +517,7 @@ static mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, long // We freed a block in an abandoned page (that was not owned). Try to collect -static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free, bool allow_reclaim, size_t counter) mi_attr_noexcept +static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free, bool allow_reclaim) mi_attr_noexcept { mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(mi_page_is_abandoned(page)); @@ -558,7 +551,7 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* if (mi_abandoned_page_try_reabandon_to_mapped(page)) return; // otherwise unown the page again - mi_abandoned_page_unown_from_free(page, mt_free, counter); + mi_abandoned_page_unown_from_free(page, mt_free); } diff --git a/src/page.c b/src/page.c index f26657788..aa669cdd0 100644 --- a/src/page.c +++ b/src/page.c @@ -363,7 +363,7 @@ static void mi_page_thread_free_collect(mi_page_t* page) do { head = mi_tf_block(tfree); if mi_likely(head == NULL) return; // return if the list is empty - tfreex = mi_tf_create(NULL,mi_tf_is_owned(tfree),1); // set the thread free list to NULL + tfreex = mi_tf_create(NULL,mi_tf_is_owned(tfree)); // set the thread free list to NULL } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tfree, tfreex)); // release is enough? mi_assert_internal(head != NULL); From 5e4b2ceadf480c4e98acf98029f33ab4c8a7ff97 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 15 Sep 2026 17:34:31 -0700 Subject: [PATCH 15/24] revise once more the non-null theap allocation variants; reduce cases and remove the xtheap variants --- include/mimalloc/internal.h | 8 +- src/alloc-aligned.c | 118 +++++++------- src/alloc.c | 301 ++++++++++++++++++------------------ 3 files changed, 215 insertions(+), 212 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 38973f185..e17e23e9d 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -270,8 +270,8 @@ mi_page_t* _mi_safe_ptr_page(const void* p); void _mi_page_map_unsafe_destroy(void); // "page.c" -void* _mi_malloc_generic(mi_theap_t* xtheap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; -void* _mi_malloc_generic_no_sample(mi_theap_t* xtheap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; +void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; +void* _mi_malloc_generic_no_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks void _mi_page_unfull(mi_page_t* page); @@ -323,8 +323,8 @@ mi_msecs_t _mi_clock_start(void); // "alloc.c" void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` -void* _mi_xtheap_malloc_zero(mi_theap_t* xtheap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` -void* _mi_xtheap_realloc_zero(mi_theap_t* xtheap, void* p, size_t newsize, bool zero) mi_attr_noexcept; +void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` +void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero) mi_attr_noexcept; mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p); void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size); diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index bdd7f626b..51f29732c 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -51,7 +51,7 @@ static mi_decl_noinline mi_decl_restrict void* mi_theap_malloc_guarded_aligned(m // Fallback aligned allocation that over-allocates -- split out for better codegen -static mi_decl_noinline void* mi_xtheap_malloc_zero_aligned_at_overalloc(mi_theap_t* const xtheap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept +static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept { mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)); mi_assert_internal(mi_alignment_is_valid(alignment)); @@ -70,7 +70,7 @@ static mi_decl_noinline void* mi_xtheap_malloc_zero_aligned_at_overalloc(mi_thea } oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size); // note: no guarded as alignment > 0 - p = _mi_xtheap_malloc_zero(xtheap, oversize, zero, alignment, &page); // the page block size should be large enough to align in the single huge page block + p = _mi_theap_malloc_zero(theap, oversize, zero, alignment, &page); // the page block size should be large enough to align in the single huge page block if (p == NULL) return NULL; } else { @@ -78,7 +78,7 @@ static mi_decl_noinline void* mi_xtheap_malloc_zero_aligned_at_overalloc(mi_thea mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) && alignment <= MI_PAGE_MAX_OVERALLOC_ALIGN); mi_assert_internal(size < SIZE_MAX - alignment); // `oversize` cannot overflow oversize = (size < MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : size) + alignment - 1; // adjust for size <= 16; with size 0 and alignment 64k, we would allocate a 64k block and pointing just beyond that. - p = _mi_xtheap_malloc_zero(xtheap, oversize, zero, 0, &page); + p = _mi_theap_malloc_zero(theap, oversize, zero, 0, &page); if (p == NULL) return NULL; } mi_assert_internal(page == _mi_ptr_page(p)); @@ -132,7 +132,7 @@ static mi_decl_noinline void* mi_xtheap_malloc_zero_aligned_at_overalloc(mi_thea } // Generic primitive aligned allocation -- split out for better codegen -static mi_decl_noinline void* mi_xtheap_malloc_zero_aligned_at_generic(mi_theap_t* const xtheap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept +static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept { mi_assert_internal(mi_alignment_is_valid(alignment)); // we don't allocate more than MI_MAX_ALLOC_SIZE (see ) @@ -145,17 +145,17 @@ static mi_decl_noinline void* mi_xtheap_malloc_zero_aligned_at_generic(mi_theap_ // this is important to try as the fast path in `mi_theap_malloc_zero_aligned` only works when there exist // a page with the right block size, and if we always use the over-alloc fallback that would never happen. #if MI_THEAP_INITASNULL - if mi_likely(xtheap!=NULL) + if mi_likely(theap!=NULL) #endif { #if MI_SAMPLE // only try if we would not take a sample - if mi_likely(!mi_theap_should_sample(xtheap,size)) + if mi_likely(!mi_theap_should_sample(theap,size)) #endif { if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) { mi_page_t* page = NULL; - void* p = _mi_xtheap_malloc_zero(xtheap, size, zero, 0,&page); + void* p = _mi_theap_malloc_zero(theap, size, zero, 0,&page); if (ppage!=NULL) { *ppage = page; } const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0; if mi_likely(is_aligned_or_null) { @@ -172,7 +172,7 @@ static mi_decl_noinline void* mi_xtheap_malloc_zero_aligned_at_generic(mi_theap_ } // fall back to over-allocation - return mi_xtheap_malloc_zero_aligned_at_overalloc(xtheap,size,alignment,offset,zero,ppage); + return mi_theap_malloc_zero_aligned_at_overalloc(theap,size,alignment,offset,zero,ppage); } @@ -182,7 +182,7 @@ static mi_decl_cold mi_decl_noinline void* mi_error_bad_alignment(size_t size, s } // Primitive aligned allocation -static inline void* mi_xtheap_malloc_zero_aligned_at(mi_theap_t* const xtheap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept +static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept { // note: we don't require `size > offset`, we just guarantee that the address at offset is aligned regardless of the allocated size. if mi_unlikely(!mi_alignment_is_valid(alignment)) { // require power-of-two and multiple of void* (see ) @@ -195,22 +195,22 @@ static inline void* mi_xtheap_malloc_zero_aligned_at(mi_theap_t* const xtheap, c if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size) { #if MI_THEAP_INITASNULL - if mi_likely(xtheap!=NULL) + if mi_likely(theap!=NULL) #endif { #if MI_SAMPLE // check if we shouldn't take a sample - if mi_likely(!mi_theap_should_sample(xtheap,size)) + if mi_likely(!mi_theap_should_sample(theap,size)) #endif { const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` const size_t padsize = size + MI_PADDING_SIZE; - mi_page_t* page = _mi_theap_get_free_small_page(xtheap, padsize, false); + mi_page_t* page = _mi_theap_get_free_small_page(theap, padsize, false); if mi_likely(page->free != NULL) { const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0; if mi_likely(is_aligned) { if (ppage!=NULL) { *ppage = page; } - void* p = _mi_page_malloc_zero(xtheap, page, padsize, zero); // xtheap!=NULL + void* p = _mi_page_malloc_zero(theap, page, padsize, zero); // theap!=NULL mi_assert_internal(p != NULL); mi_assert_internal(((uintptr_t)p + offset) % alignment == 0); mi_track_malloc(p, size, zero); @@ -222,7 +222,7 @@ static inline void* mi_xtheap_malloc_zero_aligned_at(mi_theap_t* const xtheap, c } // fallback to generic aligned allocation - return mi_xtheap_malloc_zero_aligned_at_generic(xtheap, size, alignment, offset, zero, ppage); + return mi_theap_malloc_zero_aligned_at_generic(theap, size, alignment, offset, zero, ppage); } @@ -230,30 +230,30 @@ static inline void* mi_xtheap_malloc_zero_aligned_at(mi_theap_t* const xtheap, c // Internal mi_theap_malloc_aligned / mi_malloc_aligned // ------------------------------------------------------ -static mi_decl_restrict void* mi_theap_malloc_aligned_at(mi_theap_t* xtheap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_xtheap_malloc_zero_aligned_at(xtheap, size, alignment, offset, false, NULL); +static mi_decl_restrict void* mi_theap_malloc_aligned_at(mi_theap_t* theap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_theap_malloc_zero_aligned_at(theap, size, alignment, offset, false, NULL); } -mi_decl_nodiscard mi_decl_restrict void* mi_theap_malloc_aligned(mi_theap_t* xtheap, size_t size, size_t alignment) mi_attr_noexcept { - return mi_theap_malloc_aligned_at(xtheap, size, alignment, 0); +mi_decl_nodiscard mi_decl_restrict void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept { + return mi_theap_malloc_aligned_at(theap, size, alignment, 0); } -static mi_decl_restrict void* mi_theap_zalloc_aligned_at(mi_theap_t* xtheap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_xtheap_malloc_zero_aligned_at(xtheap, size, alignment, offset, true, NULL); +static mi_decl_restrict void* mi_theap_zalloc_aligned_at(mi_theap_t* theap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_theap_malloc_zero_aligned_at(theap, size, alignment, offset, true, NULL); } -mi_decl_restrict void* mi_theap_zalloc_aligned(mi_theap_t* xtheap, size_t size, size_t alignment) mi_attr_noexcept { - return mi_theap_zalloc_aligned_at(xtheap, size, alignment, 0); +mi_decl_restrict void* mi_theap_zalloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept { + return mi_theap_zalloc_aligned_at(theap, size, alignment, 0); } -static mi_decl_restrict void* mi_theap_calloc_aligned_at(mi_theap_t* xtheap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { +static mi_decl_restrict void* mi_theap_calloc_aligned_at(mi_theap_t* theap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { size_t total; if (mi_count_size_overflow(count, size, &total)) return NULL; - return mi_theap_zalloc_aligned_at(xtheap, total, alignment, offset); + return mi_theap_zalloc_aligned_at(theap, total, alignment, offset); } -static mi_decl_restrict void* mi_theap_calloc_aligned(mi_theap_t* xtheap, size_t count, size_t size, size_t alignment) mi_attr_noexcept { - return mi_theap_calloc_aligned_at(xtheap, count, size, alignment, 0); +static mi_decl_restrict void* mi_theap_calloc_aligned(mi_theap_t* theap, size_t count, size_t size, size_t alignment) mi_attr_noexcept { + return mi_theap_calloc_aligned_at(theap, count, size, alignment, 0); } @@ -271,7 +271,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t a mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = mi_xtheap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, false, &page); + void* p = mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, false, &page); if (p!=NULL && pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); } return p; } @@ -286,7 +286,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t a mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = mi_xtheap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, true, &page); + void* p = mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, true, &page); if (p!=NULL && pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); } return p; } @@ -329,13 +329,13 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, // Aligned re-allocation // ------------------------------------------------------ -static void* mi_xtheap_realloc_zero_aligned_at(mi_theap_t* xtheap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero) mi_attr_noexcept { +static void* mi_theap_realloc_zero_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero) mi_attr_noexcept { mi_assert(mi_alignment_is_valid(alignment)); if mi_unlikely(!mi_alignment_is_valid(alignment)) { // require power-of-two (see ) return mi_error_bad_alignment(newsize,alignment,offset); } - if (alignment <= sizeof(uintptr_t) && offset==0) return _mi_xtheap_realloc_zero(xtheap,p,newsize,zero); - if (p == NULL) return mi_xtheap_malloc_zero_aligned_at(xtheap,newsize,alignment,offset,zero,NULL); + if (alignment <= sizeof(uintptr_t) && offset==0) return _mi_theap_realloc_zero(theap,p,newsize,zero); + if (p == NULL) return mi_theap_malloc_zero_aligned_at(theap,newsize,alignment,offset,zero,NULL); const size_t size = mi_usable_size(p); if (newsize <= size && newsize >= (size - (size / 2)) && (((uintptr_t)p + offset) & (alignment-1)) == 0) { return p; // reallocation still fits, is aligned and not more than 50% waste @@ -343,7 +343,7 @@ static void* mi_xtheap_realloc_zero_aligned_at(mi_theap_t* xtheap, void* p, size else { // note: we don't zero allocate upfront so we only zero initialize the expanded part (at the cost of calling mi_usable_size) mi_page_t* newpage; - void* const newp = mi_xtheap_malloc_zero_aligned_at(xtheap,newsize,alignment,offset,false/*zero?*/,&newpage); + void* const newp = mi_theap_malloc_zero_aligned_at(theap,newsize,alignment,offset,false/*zero?*/,&newpage); if (newp != NULL) { const size_t copy_size = (newsize > size ? size : newsize); const size_t zero_start = (copy_size >= sizeof(intptr_t) ? copy_size - sizeof(intptr_t) : 0); // also set last word in the previous allocation to zero to ensure any padding is zero-initialized @@ -360,88 +360,88 @@ static void* mi_xtheap_realloc_zero_aligned_at(mi_theap_t* xtheap, void* p, size } } -static void* mi_xtheap_realloc_zero_aligned(mi_theap_t* xtheap, void* p, size_t newsize, size_t alignment, bool zero) mi_attr_noexcept { +static void* mi_theap_realloc_zero_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, bool zero) mi_attr_noexcept { mi_assert(alignment > 0); - if (alignment <= sizeof(uintptr_t)) return _mi_xtheap_realloc_zero(xtheap,p,newsize,zero); - return mi_xtheap_realloc_zero_aligned_at(xtheap,p,newsize,alignment,0,zero); + if (alignment <= sizeof(uintptr_t)) return _mi_theap_realloc_zero(theap,p,newsize,zero); + return mi_theap_realloc_zero_aligned_at(theap,p,newsize,alignment,0,zero); } -static void* mi_xtheap_realloc_aligned_at(mi_theap_t* xtheap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_xtheap_realloc_zero_aligned_at(xtheap,p,newsize,alignment,offset,false); +static void* mi_theap_realloc_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_theap_realloc_zero_aligned_at(theap,p,newsize,alignment,offset,false); } -static void* mi_xtheap_realloc_aligned(mi_theap_t* xtheap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { - return mi_xtheap_realloc_zero_aligned(xtheap,p,newsize,alignment,false); +static void* mi_theap_realloc_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { + return mi_theap_realloc_zero_aligned(theap,p,newsize,alignment,false); } -static void* mi_xtheap_rezalloc_aligned_at(mi_theap_t* xtheap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_xtheap_realloc_zero_aligned_at(xtheap, p, newsize, alignment, offset, true); +static void* mi_theap_rezalloc_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_theap_realloc_zero_aligned_at(theap, p, newsize, alignment, offset, true); } -static void* mi_xtheap_rezalloc_aligned(mi_theap_t* xtheap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { - return mi_xtheap_realloc_zero_aligned(xtheap, p, newsize, alignment, true); +static void* mi_theap_rezalloc_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { + return mi_theap_realloc_zero_aligned(theap, p, newsize, alignment, true); } -static void* mi_xtheap_recalloc_aligned_at(mi_theap_t* xtheap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { +static void* mi_theap_recalloc_aligned_at(mi_theap_t* theap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { size_t total; if (mi_count_size_overflow(newcount, size, &total)) return NULL; - return mi_xtheap_rezalloc_aligned_at(xtheap, p, total, alignment, offset); + return mi_theap_rezalloc_aligned_at(theap, p, total, alignment, offset); } -static void* mi_xtheap_recalloc_aligned(mi_theap_t* xtheap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { +static void* mi_theap_recalloc_aligned(mi_theap_t* theap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { size_t total; if (mi_count_size_overflow(newcount, size, &total)) return NULL; - return mi_xtheap_rezalloc_aligned(xtheap, p, total, alignment); + return mi_theap_rezalloc_aligned(theap, p, total, alignment); } mi_decl_nodiscard void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_xtheap_realloc_aligned_at(_mi_theap_default(), p, newsize, alignment, offset); + return mi_theap_realloc_aligned_at(_mi_theap_default(), p, newsize, alignment, offset); } mi_decl_nodiscard void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept { - return mi_xtheap_realloc_aligned(_mi_theap_default(), p, newsize, alignment); + return mi_theap_realloc_aligned(_mi_theap_default(), p, newsize, alignment); } mi_decl_nodiscard void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_xtheap_rezalloc_aligned_at(_mi_theap_default(), p, newsize, alignment, offset); + return mi_theap_rezalloc_aligned_at(_mi_theap_default(), p, newsize, alignment, offset); } mi_decl_nodiscard void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept { - return mi_xtheap_rezalloc_aligned(_mi_theap_default(), p, newsize, alignment); + return mi_theap_rezalloc_aligned(_mi_theap_default(), p, newsize, alignment); } mi_decl_nodiscard void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_xtheap_recalloc_aligned_at(_mi_theap_default(), p, newcount, size, alignment, offset); + return mi_theap_recalloc_aligned_at(_mi_theap_default(), p, newcount, size, alignment, offset); } mi_decl_nodiscard void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { - return mi_xtheap_recalloc_aligned(_mi_theap_default(), p, newcount, size, alignment); + return mi_theap_recalloc_aligned(_mi_theap_default(), p, newcount, size, alignment); } mi_decl_nodiscard void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_xtheap_realloc_aligned_at(_mi_heap_theap(heap), p, newsize, alignment, offset); + return mi_theap_realloc_aligned_at(_mi_heap_theap(heap), p, newsize, alignment, offset); } mi_decl_nodiscard void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { - return mi_xtheap_realloc_aligned(_mi_heap_theap(heap), p, newsize, alignment); + return mi_theap_realloc_aligned(_mi_heap_theap(heap), p, newsize, alignment); } mi_decl_nodiscard void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_xtheap_rezalloc_aligned_at(_mi_heap_theap(heap), p, newsize, alignment, offset); + return mi_theap_rezalloc_aligned_at(_mi_heap_theap(heap), p, newsize, alignment, offset); } mi_decl_nodiscard void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { - return mi_xtheap_rezalloc_aligned(_mi_heap_theap(heap), p, newsize, alignment); + return mi_theap_rezalloc_aligned(_mi_heap_theap(heap), p, newsize, alignment); } mi_decl_nodiscard void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_xtheap_recalloc_aligned_at(_mi_heap_theap(heap), p, newcount, size, alignment, offset); + return mi_theap_recalloc_aligned_at(_mi_heap_theap(heap), p, newcount, size, alignment, offset); } mi_decl_nodiscard void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { - return mi_xtheap_recalloc_aligned(_mi_heap_theap(heap), p, newcount, size, alignment); + return mi_theap_recalloc_aligned(_mi_heap_theap(heap), p, newcount, size, alignment); } diff --git a/src/alloc.c b/src/alloc.c index 9b8f5d97e..f276be3d3 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -21,8 +21,10 @@ terms of the MIT license. A copy of the license can be found in the file #include "free.c" #undef MI_IN_ALLOC_C + // ------------------------------------------------------ -// Allocation +// Core allocation functions. +// Every allocation goes through `mi_page_malloc_zero` // ------------------------------------------------------ #if MI_PADDING @@ -46,7 +48,8 @@ static mi_decl_noinline void mi_page_block_setup_padding(mi_page_t* page, mi_blo // Fast allocation in a page: just pop from the free list. // Fall back to generic allocation only if the list is empty. -// Note: in release mode the (inlined) routine is about 7 instructions with a single test. +// Note: even though there is a lot of checks etc in the source, +// in release mode the (inlined) routine is about 7 instructions with a single test. static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, size_t sample_countdown, bool zero, mi_page_t** ppage) mi_attr_noexcept { if (page->block_size != 0) { // not the empty theap @@ -143,8 +146,11 @@ extern void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t siz return mi_page_malloc_zero(theap, page, size, theap->sample_countdown, zero, NULL); } -// internal small size allocation -static mi_decl_forceinline mi_decl_restrict void* mi_theap_xmalloc_small_zero(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept + +// Internal small size allocation; assumes the `theap` is non-NULL. +// This looks up the page directly from a the direct page entries in the theap. +// The size can be in words (as a wsize); we assume this will be inlined for best code for `mi_wmalloc_small` and similar functions. +static mi_decl_forceinline mi_decl_restrict void* mi_theap_nonnull_xmalloc_small_zero(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept { size_t size = (is_wsize ? xsize * MI_SIZE_SIZE : xsize); mi_assert(theap != NULL); @@ -176,16 +182,17 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_xmalloc_small_zero(mi return p; } -// internal generic allocation -// we use mi_xtheap_ to signify that the `theap` can be NULL (on Windows and macOS) -static mi_decl_forceinline void* mi_xtheap_malloc_generic(mi_theap_t* xtheap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept +// Internal generic allocation. +// Goes to `page.c:_mi_malloc_generic` which handles large and aligned allocations. +// Every small allocation eventually goes through this generic allocation path if it cannot be satisfied from the small page cache. +static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { #if !MI_THEAP_INITASNULL - mi_assert(xtheap!=NULL); + mi_assert(theap!=NULL); #endif - mi_assert(mi_theap_matches_thread(xtheap)); // theaps are thread local + mi_assert(mi_theap_matches_thread(theap)); // theaps are thread local mi_assert((huge_alignment & 1)==0); - void* const p = _mi_malloc_generic(xtheap, size + MI_PADDING_SIZE, (zero ? 1 : 0) | huge_alignment, ppage); // note: size can overflow but it is detected in malloc_generic + void* const p = _mi_malloc_generic(theap, size + MI_PADDING_SIZE, (zero ? 1 : 0) | huge_alignment, ppage); // note: size can overflow but it is detected in malloc_generic mi_track_malloc(p, size, zero); #if MI_DEBUG>3 @@ -196,192 +203,190 @@ static mi_decl_forceinline void* mi_xtheap_malloc_generic(mi_theap_t* xtheap, si return p; } - // ------------------------------------------------------------------------ -// We make extensive use of inline functions to define all allocation -// variants. The number of variants is a bit large as we like to optimize them well. -// -// For example, the `theap` variants should not have NULL -// theap pointers so we can avoid a NULL check. We use the `xtheap` -// name for theaps that can potentially be NULL. This is perhaps a bit too -// much though as it is only needed for Windows and macOS. -// We already simplify this for realloc and new variants. +// Small allocations: mi_malloc_small and variants (zalloc, wmalloc, wzalloc) // ------------------------------------------------------------------------ -// internal small allocation -static mi_decl_forceinline mi_decl_restrict void* mi_xtheap_xmalloc_small_zero(mi_theap_t* xtheap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept { - #if MI_THEAP_INITASNULL - if (xtheap!=NULL) { - return mi_theap_xmalloc_small_zero(xtheap, xsize, is_wsize, zero, ppage); +// internal small allocation where the theap can be NULL on some platforms (Windows and macOS) +static mi_decl_forceinline mi_decl_restrict void* mi_theap_xmalloc_small_zero(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept { + #if !MI_THEAP_INITASNULL + return mi_theap_nonnull_xmalloc_small_zero(theap, xsize, is_wsize, zero, ppage); + #else + if (theap!=NULL) { + return mi_theap_nonnull_xmalloc_small_zero(theap, xsize, is_wsize, zero, ppage); } else { - return mi_xtheap_malloc_generic(xtheap, (is_wsize ? xsize * MI_SIZE_SIZE : xsize), zero, 0, ppage); // tailcall + return mi_theap_malloc_generic(theap, (is_wsize ? xsize * MI_SIZE_SIZE : xsize), zero, 0, ppage); // tailcall } - #else - return mi_theap_xmalloc_small_zero(xtheap, xsize, is_wsize, zero, ppage); #endif } -// allocate a small block +// Allocate a small block from a theap; these routines assume a non-NULL theap. mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { mi_assert(theap!=NULL); - return mi_theap_xmalloc_small_zero(theap, size, false, false, NULL); + return mi_theap_nonnull_xmalloc_small_zero(theap, size, false, false, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept { mi_assert(theap!=NULL); - return mi_theap_xmalloc_small_zero(theap, wsize, true, false, NULL); + return mi_theap_nonnull_xmalloc_small_zero(theap, wsize, true, false, NULL); +} + +mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { + mi_assert(theap!=NULL); + return mi_theap_nonnull_xmalloc_small_zero(theap, size, false, true, NULL); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_theap_wzalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept { + mi_assert(theap!=NULL); + return mi_theap_nonnull_xmalloc_small_zero(theap, wsize, true, true, NULL); } +// Regular small allocation functions that use the default theap. mi_decl_nodiscard mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept { - return mi_xtheap_xmalloc_small_zero(_mi_theap_default(), size, false, false, NULL); + return mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, false, NULL); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept { + return mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, true, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_wmalloc_small(size_t wsize) mi_attr_noexcept { - return mi_xtheap_xmalloc_small_zero(_mi_theap_default(), wsize, true, false, NULL ); + return mi_theap_xmalloc_small_zero(_mi_theap_default(), wsize, true, false, NULL ); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_wzalloc_small(size_t wsize) mi_attr_noexcept { + return mi_theap_xmalloc_small_zero(_mi_theap_default(), wsize, true, true, NULL); } + mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { return mi_theap_malloc_small(_mi_heap_theap_get_or_init(heap), size); } - mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { // we could also use: mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(theap), size, false, NULL); } // but the following prevents using a stack frame. We use this to optimize some select functions only. - mi_theap_t* const xtheap = _mi_heap_theap_cached(heap); - if mi_likely(xtheap!=NULL) { return mi_theap_malloc_small(xtheap, size); } - else { return mi_heap_init_malloc_small(heap, size); } + mi_theap_t* const theap = _mi_heap_theap_cached(heap); + if mi_likely(theap!=NULL) { return mi_theap_malloc_small(theap, size); } + else { return mi_heap_init_malloc_small(heap, size); } +} + +mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return mi_theap_zalloc_small(_mi_heap_theap_get_or_init(heap), size); } +mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { + // optimize: return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, true, NULL); + mi_theap_t* const theap = _mi_heap_theap_cached(heap); + if mi_likely(theap!=NULL) { return mi_theap_zalloc_small(theap, size); } + else { return mi_heap_init_zalloc_small(heap, size); } +} + + +// ------------------------------------------------------------------------------ +// Main allocation functions: mi_malloc and variants (zalloc, wmalloc, wzalloc) +// ------------------------------------------------------------------------------ -// The main internal allocation functions -static mi_decl_forceinline void* mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { +mi_decl_nodiscard static mi_decl_forceinline mi_decl_restrict void* mi_theap_nonnull_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { mi_assert(theap!=NULL); // fast path for small objects if mi_likely(size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(huge_alignment == 0); - return mi_theap_xmalloc_small_zero(theap, size, false, zero, ppage); + return mi_theap_nonnull_xmalloc_small_zero(theap, size, false, zero, ppage); } else { - return mi_xtheap_malloc_generic(theap, size, zero, huge_alignment, ppage); + return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage); } } -extern mi_decl_forceinline void* _mi_xtheap_malloc_zero(mi_theap_t* xtheap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { +mi_decl_nodiscard extern mi_decl_forceinline mi_decl_restrict void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { // fast path for small objects #if MI_THEAP_INITASNULL - if mi_likely(xtheap!=NULL && size <= MI_SMALL_SIZE_MAX) + if mi_likely(theap!=NULL && size <= MI_SMALL_SIZE_MAX) #else if mi_likely(size <= MI_SMALL_SIZE_MAX) #endif { mi_assert_internal(huge_alignment == 0); - return mi_theap_xmalloc_small_zero(xtheap, size, false, zero, ppage); + return mi_theap_nonnull_xmalloc_small_zero(theap, size, false, zero, ppage); } else { - return mi_xtheap_malloc_generic(xtheap, size, zero, huge_alignment, ppage); + return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage); } } +mi_decl_nodiscard static inline mi_decl_restrict void* mi_theap_nonnull_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept { + mi_assert(theap!=NULL); + return mi_theap_nonnull_malloc_zero(theap, size, false, 0, NULL); +} +mi_decl_nodiscard static inline mi_decl_restrict void* mi_theap_nonnull_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept { + mi_assert(theap!=NULL); + return mi_theap_nonnull_malloc_zero(theap, size, true, 0, NULL); +} // Main allocation functions +// (note: to reduce variants, we only use nonnull theap variants for small allocations) mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept { - mi_assert(theap!=NULL); - return mi_theap_malloc_zero(theap, size, false, 0, NULL); + return _mi_theap_malloc_zero(theap, size, false, 0, NULL); } -static mi_decl_restrict void* mi_xtheap_malloc(mi_theap_t* xtheap, size_t size) mi_attr_noexcept { - return _mi_xtheap_malloc_zero(xtheap, size, false, 0, NULL); +mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept { + return _mi_theap_malloc_zero(theap, size, true, 0, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept { - return mi_xtheap_malloc(_mi_theap_default(), size); + return mi_theap_malloc(_mi_theap_default(), size); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept { + return mi_theap_zalloc(_mi_theap_default(), size); } mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_malloc(_mi_heap_theap_get_or_init(heap), size); + return mi_theap_nonnull_malloc(_mi_heap_theap_get_or_init(heap), size); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { // return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, false, 0, NULL); - mi_theap_t* const xtheap = _mi_heap_theap_cached(heap); - if mi_likely(xtheap!=NULL) { return mi_theap_malloc(xtheap, size); } - else { return mi_heap_init_malloc(heap, size); } - -} - - -// zero initialized small block -mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept { - return mi_xtheap_xmalloc_small_zero(_mi_theap_default(), size, false, true, NULL); -} - -mi_decl_nodiscard mi_decl_restrict void* mi_wzalloc_small(size_t wsize) mi_attr_noexcept { - return mi_xtheap_xmalloc_small_zero(_mi_theap_default(), wsize, true, true, NULL); -} - -mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { - mi_assert(theap!=NULL); - return mi_theap_xmalloc_small_zero(theap, size, false, true, NULL); -} + mi_theap_t* const theap = _mi_heap_theap_cached(heap); + if mi_likely(theap!=NULL) { return mi_theap_nonnull_malloc(theap, size); } + else { return mi_heap_init_malloc(heap, size); } -mi_decl_nodiscard mi_decl_restrict void* mi_theap_wzalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept { - mi_assert(theap!=NULL); - return mi_theap_xmalloc_small_zero(theap, wsize, true, true, NULL); -} - -mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_zalloc_small(_mi_heap_theap_get_or_init(heap), size); -} -mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { - // optimize: return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, true, NULL); - mi_theap_t* const xtheap = _mi_heap_theap_cached(heap); - if mi_likely(xtheap!=NULL) { return mi_theap_zalloc_small(xtheap, size); } - else { return mi_heap_init_zalloc_small(heap, size); } -} - -mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept { - mi_assert(theap!=NULL); - return mi_theap_malloc_zero(theap, size, true, 0, NULL); -} - -mi_decl_nodiscard static mi_decl_restrict void* mi_xtheap_zalloc(mi_theap_t* xtheap, size_t size) mi_attr_noexcept { - return _mi_xtheap_malloc_zero(xtheap, size, true, 0, NULL); -} - -mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept { - return _mi_xtheap_malloc_zero(_mi_theap_default(), size, true, 0, NULL); } mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_zalloc(_mi_heap_theap_get_or_init(heap), size); + return mi_theap_nonnull_zalloc(_mi_heap_theap_get_or_init(heap), size); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { // optimize: return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, true, 0, NULL); - mi_theap_t* const xtheap = _mi_heap_theap_cached(heap); - if mi_likely(xtheap!=NULL) { return mi_theap_zalloc(xtheap, size); } - else { return mi_heap_init_zalloc(heap, size); } + mi_theap_t* const theap = _mi_heap_theap_cached(heap); + if mi_likely(theap!=NULL) { return mi_theap_nonnull_zalloc(theap, size); } + else { return mi_heap_init_zalloc(heap, size); } } +// ---------------------------------------------------------- +// mi_calloc variants +// ---------------------------------------------------------- + mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept { - mi_assert(theap!=NULL); size_t total; if (mi_count_size_overflow(count,size,&total)) return NULL; return mi_theap_zalloc(theap,total); } mi_decl_nodiscard mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept { - size_t total; - if (mi_count_size_overflow(count,size,&total)) return NULL; - return mi_zalloc(total); + return mi_theap_calloc(_mi_theap_default(), count, size); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept { - size_t total; - if (mi_count_size_overflow(count, size, &total)) return NULL; - return mi_heap_zalloc(heap, total); + return mi_theap_calloc(_mi_heap_theap(heap), count, size); } +// ---------------------------------------------------------- +// Usable block sizes +// ---------------------------------------------------------- + +// Return usable size static void* mi_ublock_size( void* p, mi_page_t* page, size_t* pblock_size ) { mi_assert_internal(page == _mi_ptr_page(p)); if (pblock_size!=NULL) { @@ -390,28 +395,27 @@ static void* mi_ublock_size( void* p, mi_page_t* page, size_t* pblock_size ) { return p; } -// Return usable size mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = mi_xtheap_xmalloc_small_zero(_mi_theap_default(), size, false, false, &page); + void* p = mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, false, &page); return mi_ublock_size(p,page,pblock_size); } mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = mi_xtheap_xmalloc_small_zero(_mi_theap_default(), size, false, true, &page); + void* p = mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, true, &page); return mi_ublock_size(p,page,pblock_size); } mi_decl_nodiscard mi_decl_restrict void* mi_umalloc(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = _mi_xtheap_malloc_zero(_mi_theap_default(), size, false, 0, &page); + void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, false, 0, &page); return mi_ublock_size(p,page,pblock_size); } mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = _mi_xtheap_malloc_zero(_mi_theap_default(), size, true, 0, &page); + void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, true, 0, &page); return mi_ublock_size(p,page,pblock_size); } @@ -421,7 +425,10 @@ mi_decl_nodiscard mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, s return mi_uzalloc(total, pblock_size); } +// ---------------------------------------------------------- // Uninitialized `calloc` +// ---------------------------------------------------------- + mi_decl_nodiscard mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept { size_t total; if (mi_count_size_overflow(count, size, &total)) return NULL; @@ -434,6 +441,9 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t return mi_heap_malloc(heap, total); } +// ---------------------------------------------------------- +// Reallocation +// ---------------------------------------------------------- // Expand (or shrink) in place (or fail) void* mi_expand(void* p, size_t newsize) mi_attr_noexcept { @@ -450,7 +460,7 @@ void* mi_expand(void* p, size_t newsize) mi_attr_noexcept { #endif } -static mi_decl_forceinline void* mi_xtheap_realloc_zero_ex(mi_theap_t* xtheap, void* p, size_t newsize, bool zero, size_t* pblock_size_pre, size_t* pblock_size_post) mi_attr_noexcept { +static mi_decl_forceinline void* mi_theap_realloc_zero_ex(mi_theap_t* theap, void* p, size_t newsize, bool zero, size_t* pblock_size_pre, size_t* pblock_size_post) mi_attr_noexcept { // if p == NULL then behave as malloc. // else if size == 0 then reallocate to a zero-sized block (and don't return NULL, just as mi_malloc(0)). // (this means that returning NULL always indicates an error, and `p` will not have been freed in that case.) @@ -475,10 +485,10 @@ static mi_decl_forceinline void* mi_xtheap_realloc_zero_ex(mi_theap_t* xtheap, v if mi_unlikely(newsize<=size && newsize>=(size/2) && newsize>0) { // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0) mi_assert_internal(page!=NULL); // note: page!=NULL (since if p==NULL, we have size=0 and size>=newsize>0 #if MI_THEAP_INITASNULL - if (xtheap!=NULL) + if (theap!=NULL) #endif { - if (mi_page_heap(page)==_mi_theap_heap_peek(xtheap)) { // and within the same heap + if (mi_page_heap(page)==_mi_theap_heap_peek(theap)) { // and within the same heap mi_assert_internal(p!=NULL); // todo: do not track as the usable size is still the same in the free; adjust potential padding? // mi_track_resize(p,size,newsize) @@ -490,7 +500,7 @@ static mi_decl_forceinline void* mi_xtheap_realloc_zero_ex(mi_theap_t* xtheap, v } // note: we don't zero allocate upfront so we only zero initialize the expanded part mi_page_t* newpage; // use block_size for zero-ing, issue #763 - void* const newp = _mi_xtheap_malloc_zero(xtheap,newsize,false /* no zero */,0,&newpage); + void* const newp = _mi_theap_malloc_zero(theap,newsize,false /* no zero */,0,&newpage); if mi_likely(newp != NULL) { if (pblock_size_post!=NULL) { *pblock_size_post = mi_page_block_size(newpage); } const size_t copy_size = (newsize > size ? size : newsize); @@ -512,33 +522,33 @@ static mi_decl_forceinline void* mi_xtheap_realloc_zero_ex(mi_theap_t* xtheap, v return newp; } -void* _mi_xtheap_realloc_zero(mi_theap_t* xtheap, void* p, size_t newsize, bool zero) mi_attr_noexcept { - return mi_xtheap_realloc_zero_ex(xtheap,p,newsize,zero,NULL,NULL); +void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero) mi_attr_noexcept { + return mi_theap_realloc_zero_ex(theap,p,newsize,zero,NULL,NULL); } // for theap_realloc/rezalloc we allow theap==NULL to reduce variants -mi_decl_nodiscard void* mi_theap_realloc(mi_theap_t* xtheap, void* p, size_t newsize) mi_attr_noexcept { +mi_decl_nodiscard void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept { if (p==NULL) { - return mi_xtheap_malloc(xtheap,newsize); + return mi_theap_malloc(theap,newsize); } else { - return _mi_xtheap_realloc_zero(xtheap, p, newsize, false); + return _mi_theap_realloc_zero(theap, p, newsize, false); } } -mi_decl_nodiscard void* mi_theap_rezalloc(mi_theap_t* xtheap, void* p, size_t newsize) mi_attr_noexcept { +mi_decl_nodiscard void* mi_theap_rezalloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept { if (p==NULL) { - return mi_xtheap_zalloc(xtheap,newsize); + return mi_theap_zalloc(theap,newsize); } else { - return _mi_xtheap_realloc_zero(xtheap, p, newsize, true); + return _mi_theap_realloc_zero(theap, p, newsize, true); } } -static void* mi_theap_recalloc(mi_theap_t* xtheap, void* p, size_t count, size_t size) mi_attr_noexcept { +static void* mi_theap_recalloc(mi_theap_t* theap, void* p, size_t count, size_t size) mi_attr_noexcept { size_t total; if (mi_count_size_overflow(count, size, &total)) return NULL; - return mi_theap_rezalloc(xtheap, p, total); + return mi_theap_rezalloc(theap, p, total); } mi_decl_nodiscard void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept { @@ -552,7 +562,7 @@ mi_decl_nodiscard void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_ } mi_decl_nodiscard void* mi_urealloc(void* p, size_t newsize, size_t* pblock_size_pre, size_t* pblock_size_post) mi_attr_noexcept { - return mi_xtheap_realloc_zero_ex(_mi_theap_default(),p,newsize, false, pblock_size_pre, pblock_size_post); + return mi_theap_realloc_zero_ex(_mi_theap_default(),p,newsize, false, pblock_size_pre, pblock_size_post); } // Reallocate but free `p` on errors @@ -574,7 +584,6 @@ mi_decl_nodiscard void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_ return mi_theap_recalloc(_mi_theap_default(), p, count, size); } - mi_decl_nodiscard void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { return mi_theap_realloc(_mi_heap_theap(heap), p, newsize); } @@ -599,7 +608,6 @@ mi_decl_nodiscard void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t count, } - // ------------------------------------------------------ // strdup, strndup, and realpath // ------------------------------------------------------ @@ -609,7 +617,7 @@ mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strdup(mi_theap_t* xthe if (s == NULL) return NULL; size_t len = _mi_strlen(s); if (len > MI_MAX_ALLOC_SIZE - 1) return NULL; // prevent overflow on len+1 - char* t = (char*)mi_xtheap_malloc(xtheap,len+1); + char* t = (char*)mi_theap_malloc(xtheap,len+1); if (t == NULL) return NULL; _mi_memcpy(t, s, len); t[len] = 0; @@ -629,7 +637,7 @@ mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strndup(mi_theap_t* xth if (s == NULL) return NULL; const size_t len = _mi_strnlen(s,n); // len <= n if (len > MI_MAX_ALLOC_SIZE - 1) return NULL; // prevent overflow on len+1 - char* t = (char*)mi_xtheap_malloc(xtheap, len+1); + char* t = (char*)mi_theap_malloc(xtheap, len+1); if (t == NULL) return NULL; _mi_memcpy(t, s, len); t[len] = 0; @@ -722,7 +730,7 @@ char* mi_theap_realpath(mi_theap_t* xtheap, const char* fname, char* resolved_na return result; */ const size_t n = mi_path_max(); - char* const buf = (char*)mi_xtheap_zalloc(xtheap,n+1); + char* const buf = (char*)mi_theap_zalloc(xtheap,n+1); if (buf == NULL) { errno = ENOMEM; return NULL; @@ -823,27 +831,22 @@ static bool mi_try_new_handler(bool nothrow) { } #endif -static mi_decl_noinline void* mi_xtheap_try_new(mi_theap_t* xtheap, size_t size, bool nothrow ) { +static mi_decl_noinline void* mi_theap_try_new(mi_theap_t* theap, size_t size, bool nothrow ) { void* p = NULL; for(int i = 0; i < MI_TRY_NEW_MAX && p == NULL && mi_try_new_handler(nothrow); i++) { if (size > MI_MAX_ALLOC_SIZE) return NULL; // call try_new_handler at least once - p = mi_xtheap_malloc(xtheap,size); + p = mi_theap_malloc(theap,size); } return p; } -// static mi_decl_noinline void* mi_try_new(size_t size, bool nothrow) { -// return mi_theap_try_new(_mi_theap_default(), size, nothrow); -// } - static mi_decl_noinline void* mi_heap_try_new(mi_heap_t* heap, size_t size, bool nothrow) { - return mi_xtheap_try_new(_mi_heap_theap(heap), size, nothrow); + return mi_theap_try_new(_mi_heap_theap(heap), size, nothrow); } -// again, to reduce variants we allow `xtheap==NULL`. -mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* xtheap, size_t size) { - void* p = mi_xtheap_malloc(xtheap,size); - if mi_unlikely(p == NULL) return mi_xtheap_try_new(xtheap, size, false); +mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* theap, size_t size) { + void* p = mi_theap_malloc(theap,size); + if mi_unlikely(p == NULL) return mi_theap_try_new(theap, size, false); return p; } @@ -857,14 +860,14 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size return p; } -mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* xtheap, size_t count, size_t size) { +mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* theap, size_t count, size_t size) { size_t total; if mi_unlikely(mi_count_size_overflow(count, size, &total)) { mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc return NULL; } else { - return mi_theap_alloc_new(xtheap,total); + return mi_theap_alloc_new(theap,total); } } @@ -877,9 +880,9 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, si } -mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new_nothrow(mi_theap_t* xtheap, size_t size) mi_attr_noexcept { - void* p = mi_xtheap_malloc(xtheap,size); - if mi_unlikely(p == NULL) return mi_xtheap_try_new(xtheap, size, true); +mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new_nothrow(mi_theap_t* theap, size_t size) mi_attr_noexcept { + void* p = mi_theap_malloc(theap,size); + if mi_unlikely(p == NULL) return mi_theap_try_new(theap, size, true); return p; } @@ -941,7 +944,7 @@ mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) { #ifdef __cplusplus void* _mi_externs[] = { (void*)&_mi_page_malloc_zero, - (void*)&_mi_xtheap_malloc_zero, + (void*)&_mi_theap_malloc_zero, (void*)&mi_theap_malloc, (void*)&mi_theap_zalloc, (void*)&mi_theap_malloc_small, From 014be9ac46a8fab538ff201714d6f2663f8696d3 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 15 Sep 2026 17:53:01 -0700 Subject: [PATCH 16/24] disable MEMZERO16X for now on riscV (due to compiler errors) --- include/mimalloc/internal.h | 3 ++- src/alloc.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index e17e23e9d..b79649c72 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1523,8 +1523,9 @@ static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize // fast memzero for small sizes based on overlapping writes (and assuming non-zero size_t-multiple size, and size_t aligned) // assumes constant memset(p,0,N) gets optimized to fast simd stores by the compiler + // note: disabled on riscv for now as a constant memset is not always replaced correctly by current compilers. // (compile with -DMI_USE_MEMZERO16X=0 to disable this) - #if !defined(MI_USE_MEMZERO16X) || (MI_USE_MEMZERO16X != 0) // 16x MI_SIZE_SIZE (128 bytes on 64-bit) + #if (!defined(MI_USE_MEMZERO16X) && !MI_ARCH_RISCV) || (MI_USE_MEMZERO16X != 0) // 16x MI_SIZE_SIZE (128 bytes on 64-bit) if mi_unlikely(bsize < 2*MI_SIZE_SIZE) { // bsize < 16 (8) *((size_t*)dst) = 0; return dst; diff --git a/src/alloc.c b/src/alloc.c index f276be3d3..a176db77f 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -49,7 +49,7 @@ static mi_decl_noinline void mi_page_block_setup_padding(mi_page_t* page, mi_blo // Fast allocation in a page: just pop from the free list. // Fall back to generic allocation only if the list is empty. // Note: even though there is a lot of checks etc in the source, -// in release mode the (inlined) routine is about 7 instructions with a single test. +// in release mode the (inlined) routine is about 7 to 10 instructions with a single test. static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, size_t sample_countdown, bool zero, mi_page_t** ppage) mi_attr_noexcept { if (page->block_size != 0) { // not the empty theap From d47c4ca5f6e5a7a3e96763b556346c8e2d728684 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 16 Sep 2026 15:14:24 -0700 Subject: [PATCH 17/24] bump version v3.5.3 --- .github/workflows/release.yaml | 4 ++-- cmake/mimalloc-config-version.cmake | 2 +- include/mimalloc.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 448a7ff3b..e0c9d84a9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,7 +9,7 @@ permissions: contents: write env: - RELEASE: Release v3.5.2 + RELEASE: Release v3.5.3 FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true name: Release @@ -19,7 +19,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - branch: [v3.5.2, v2.5.2, v1.15.2] # [dev,dev2,dev3] + branch: [v3.5.3, v2.5.2, v1.15.2] # [dev,dev2,dev3] # we build on the oldest ubuntu version for better binary compatibility. os: [windows-latest, macOS-latest, macos-15-intel, ubuntu-22.04, ubuntu-22.04-arm] diff --git a/cmake/mimalloc-config-version.cmake b/cmake/mimalloc-config-version.cmake index 349535cce..36d441103 100644 --- a/cmake/mimalloc-config-version.cmake +++ b/cmake/mimalloc-config-version.cmake @@ -1,6 +1,6 @@ set(mi_version_major 3) set(mi_version_minor 5) -set(mi_version_patch 2) +set(mi_version_patch 3) set(mi_version ${mi_version_major}.${mi_version_minor}) set(PACKAGE_VERSION ${mi_version}) diff --git a/include/mimalloc.h b/include/mimalloc.h index 9a97d69ec..2985edd94 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 30502 // major + 2 digits minor + 2 digits patch +#define MI_MALLOC_VERSION 30503 // major + 2 digits minor + 2 digits patch // ------------------------------------------------------ // Compiler specific attributes From 447771491f7016c001c386d729bdbf6fe0182bae Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 16 Sep 2026 15:25:57 -0700 Subject: [PATCH 18/24] fix bitmap ccount calculation --- src/bitmap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bitmap.c b/src/bitmap.c index 01de1999a..238c4d4c4 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -1081,7 +1081,7 @@ static void mi_bchunks_unsafe_setN(mi_bchunk_t* chunks, mi_bchunkmap_t* cmap, si // start chunk and index size_t chunk_idx = idx / MI_BCHUNK_BITS; const size_t cidx = idx % MI_BCHUNK_BITS; - const size_t ccount = _mi_divide_up(n, MI_BCHUNK_BITS); + const size_t ccount = ((cidx + n - 1) / MI_BCHUNK_BITS) + 1; // first update the chunkmap mi_bchunk_setN(cmap, chunk_idx, ccount, NULL); From 8352ee2a8505c1912818af61f0c262d668a3d23e Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 16 Sep 2026 15:27:03 -0700 Subject: [PATCH 19/24] update readme --- readme.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 8320d5b20..258b01550 100644 --- a/readme.md +++ b/readme.md @@ -15,7 +15,7 @@ is a general purpose allocator with excellent [performance](#performance) charac Initially developed by Daan Leijen for the runtime systems of the [Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages. -Latest release : `v3.5.2` (2026-09-12) recommended. +Latest release : `v3.5.3` (2026-09-16) recommended. Latest v2 release: `v2.5.2` (2026-09-12) stable, legacy. Latest v1 release: `v1.15.2` (2026-09-12) legacy. @@ -90,6 +90,8 @@ New development is mostly on v3, while v1 and v2 are maintained with security an ### Releases +* 2026-09-16, `v3.5.3`: (v3) critical bug fix (where the first new on a thread could + fail on some platforms). Fix bug with `mi_free_size` on overaligned small allocations. * 2026-09-12, `v3.5.2`, `v2.5.2`, `v1.15.2`: (v3) Reduced cache contention, improved `mi_malloc_csize`, improved zero'ing of small blocks, `mi_wmalloc` variants for runtime systems, always enable detailed statistics. Experimental support for profiling hooks (`mimalloc-profile.h`). From 8c377a934ba22cff863e1c1d407605bc2f4798bb Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 16 Sep 2026 15:41:22 -0700 Subject: [PATCH 20/24] fix msbuild multi config --- CMakeLists.txt | 20 +++++++++++++++++++- readme.md | 2 +- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c1431a725..a4a260af0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -931,6 +931,8 @@ endif() if(MI_TRACK STREQUAL "ASAN") set(mi_libname "${mi_libname}-asan") endif() +set(mi_libname_base "${mi_libname}") # capture the base name (without any per-config suffix) + if (CMAKE_CONFIGURATION_TYPES) # multi-config builds # generator expression lower-case build type @@ -1008,7 +1010,23 @@ if(MI_BUILD_SHARED) # so we postfix the dll import library with `.dll.lib` (and also the .pdb debug file) set_property(TARGET mimalloc PROPERTY ARCHIVE_OUTPUT_NAME "${mi_libname}.dll" ) install(FILES "$/${mi_libname}.dll.lib" DESTINATION ${CMAKE_INSTALL_LIBDIR}/mimalloc-${mi_version}) - set_property(TARGET mimalloc PROPERTY PDB_NAME "${mi_libname}.dll") + if (CMAKE_CONFIGURATION_TYPES) + # PDB_NAME only supports generator expressions since CMake 4.1; for multi-config + # generators (like Visual Studio) we instead set `PDB_NAME_` for each + # known configuration explicitly (mirroring the release/debug suffix logic above). + foreach(mi_cfg IN LISTS CMAKE_CONFIGURATION_TYPES) + string(TOUPPER "${mi_cfg}" mi_cfg_uc) + string(TOLOWER "${mi_cfg}" mi_cfg_lc) + if (mi_cfg_lc MATCHES "^(release|relwithdebinfo|minsizerel|none)$") + set(mi_libname_pdb "${mi_libname_base}") + else() + set(mi_libname_pdb "${mi_libname_base}-${mi_cfg_lc}") + endif() + set_property(TARGET mimalloc PROPERTY "PDB_NAME_${mi_cfg_uc}" "${mi_libname_pdb}.dll") + endforeach() + else() + set_property(TARGET mimalloc PROPERTY PDB_NAME "${mi_libname}.dll") + endif() # don't try to install the pdb since it may not be generated depending on the configuration # install(FILES "$/${mi_libname}.dll.pdb" DESTINATION ${CMAKE_INSTALL_LIBDIR}) endif() diff --git a/readme.md b/readme.md index 258b01550..97c1983b5 100644 --- a/readme.md +++ b/readme.md @@ -91,7 +91,7 @@ New development is mostly on v3, while v1 and v2 are maintained with security an ### Releases * 2026-09-16, `v3.5.3`: (v3) critical bug fix (where the first new on a thread could - fail on some platforms). Fix bug with `mi_free_size` on overaligned small allocations. + fail on some platforms #1398). Fix bug with `mi_free_size` on overaligned small allocations (#1400). * 2026-09-12, `v3.5.2`, `v2.5.2`, `v1.15.2`: (v3) Reduced cache contention, improved `mi_malloc_csize`, improved zero'ing of small blocks, `mi_wmalloc` variants for runtime systems, always enable detailed statistics. Experimental support for profiling hooks (`mimalloc-profile.h`). From a2d339dba141ccde89b772e3e11fdee2a5893165 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 16 Sep 2026 15:58:45 -0700 Subject: [PATCH 21/24] add extra heap alignment test --- src/free.c | 18 +++++++++--------- test/test-api.c | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/free.c b/src/free.c index 58899ecba..722ee7bf6 100644 --- a/src/free.c +++ b/src/free.c @@ -347,15 +347,15 @@ void mi_free_size(void* p, size_t size) mi_attr_noexcept { return; } } - const size_t is_aligned = ((void*)block != p); - if mi_unlikely(size <= MI_SMALL_SIZE_MAX && mi_page_block_size(page) > mi_good_size((is_aligned ? 2 : 1)*MI_SMALL_SIZE_MAX)) { // note: we check *2 in case it was over-aligned - const bool is_guarded = mi_block_ptr_is_guarded(block,p); - if (!is_guarded) { - _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the given size %zu is less than the allocated block size %zu\n (maybe a `new[]` was matched with `delete` instead of `delete[]`?)\n", p, size, mi_page_block_size(page)); - mi_free(p); - return; - } - } + // const size_t is_aligned = ((void*)block != p); + // if mi_unlikely(size <= MI_SMALL_SIZE_MAX && mi_page_block_size(page) > mi_good_size((is_aligned ? 2 : 1)*MI_SMALL_SIZE_MAX)) { // note: we check *2 in case it was over-aligned + // const bool is_guarded = mi_block_ptr_is_guarded(block,p); + // if (!is_guarded) { + // _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the given size %zu is less than the allocated block size %zu\n (maybe a `new[]` was matched with `delete` instead of `delete[]`?)\n", p, size, mi_page_block_size(page)); + // mi_free(p); + // return; + // } + // } #endif #if MI_PAGE_META_SMALL_IS_ALIGNED || MI_PAGE_META_IS_ALIGNED if mi_likely(size <= MI_SMALL_SIZE_MAX) { diff --git a/test/test-api.c b/test/test-api.c index de52d642d..abbeb434e 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -385,6 +385,20 @@ int main(void) { mi_free(ptr); } + CHECK_BODY("heap_aligned1") { + mi_heap_t* heap = mi_heap_new(); + const size_t alignment = 128 * 1024; // 128 KiB + const size_t buffer_size = 1 * 1024 * 1024; // 1 MiB + void* buffer = mi_heap_malloc_aligned(heap, buffer_size, alignment); + bool nonnull = (buffer != NULL); + assert(nonnull); + const bool is_aligned = ((uintptr_t)buffer % alignment) == 0; + assert(is_aligned); + mi_free(buffer); + mi_heap_destroy(heap); + result = (nonnull && is_aligned); + } + // --------------------------------------------------- // Reallocation // --------------------------------------------------- From 326d2719fa27d6fc16b84bc7178842f78b93d33e Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 16 Sep 2026 16:04:20 -0700 Subject: [PATCH 22/24] update readme --- readme.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/readme.md b/readme.md index 97c1983b5..eb11d7669 100644 --- a/readme.md +++ b/readme.md @@ -602,14 +602,15 @@ In the source code: - Use `mi_free_small` when the pointer was returned from `mi_malloc_small` (and guaranteed to have a size of less than `MI_SMALL_SIZE_MAX`). Use `mi_free_small_nonnull` when the pointer is guaranteed to be not `NULL` as well. -For run-time systems and compilers (like Koka, Lean, etc.), we can do slightly better still. +For run-time systems and compilers (like Koka, Lean, Rust (?) etc.), we can do slightly better still. - If the allocation size is statically known, use `mi_malloc_csize` and `mi_free_csize` etc. when possible. (or directly `mi_malloc_small`/`mi_free_small`). - If the runtime already carries thread local state, it may be faster to get the default `theap` for each thread - up-front (`mi_theap_get_default()`) and use the very fast `mi_theap_malloc(_small)` etc. passing the theap - pointer directly. This avoids having mimalloc look up the thread local pointer all the time. Whether this is + up-front (`mi_theap_get_default()`) and use the bestest `mi_theap_malloc(_small)` etc. passing the theap pointer directly. + This avoids having mimalloc look up the thread local pointer all the time and + skips a theap `NULL` check. Whether this is faster depends a bit on the OS implementation of thread local variables. From 53bbb1fcd953dc2e620d76524c8e1675472792b4 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 16 Sep 2026 16:06:28 -0700 Subject: [PATCH 23/24] update readme --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index eb11d7669..4c2ce601a 100644 --- a/readme.md +++ b/readme.md @@ -86,7 +86,7 @@ New development is mostly on v3, while v1 and v2 are maintained with security an (release tags: `v3.x`, development branch `dev3`). - __v2__: stable legacy mimalloc version. Uses thread-local segments to reduce fragmentation. (release tags: `v2.x`, development branch `dev2` and `main`) - __v1__: legacy version: initial design of mimalloc (release tags: `v1.x`, development branch `dev`). - __Send PR's against this version if possible.__ + __Send PR's against this version if possible (and otherwise `dev3`).__ ### Releases From d4881d338125e1cb7c47ba4cfb398d6f7c0c8d45 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 16 Sep 2026 17:27:08 -0700 Subject: [PATCH 24/24] fix arena allocation for exclusive child arenas, PR #1404 --- src/arena.c | 23 ++++++++++------------- test/test-api.c | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/src/arena.c b/src/arena.c index 98fb5a084..1b7c8cb33 100644 --- a/src/arena.c +++ b/src/arena.c @@ -455,24 +455,21 @@ static size_t mi_arena_start_idx(mi_heap_t* heap, size_t tseq, size_t arena_cycl const size_t _arena_count = mi_arenas_get_count(heap->subproc); \ const size_t _arena_cycle = (_arena_count == 0 ? 0 : _arena_count - 1); /* first search the arenas below the last one */ \ /* always start searching in the arena's below the max */ \ - const size_t _start = mi_arena_start_idx(heap,tseq,_arena_cycle); \ + const size_t _start = (req_arena==NULL ? mi_arena_start_idx(heap,tseq,_arena_cycle) : ((mi_arena_t*)req_arena)->arena_idx); \ + mi_assert_internal(_start <= _arena_count); \ for (size_t _i = 0; _i < _arena_count; _i++) { \ mi_arena_t* name_arena; \ - if (req_arena != NULL) { \ - name_arena = req_arena; /* if there is a specific req_arena, only search that one */\ - if (_i > 0) break; /* only once */ \ + size_t _idx; \ + if (_i < _arena_cycle) { \ + _idx = _i + _start; \ + if (_idx >= _arena_cycle) { _idx -= _arena_cycle; } /* adjust so we rotate through the cycle */ \ } \ else { \ - size_t _idx; \ - if (_i < _arena_cycle) { \ - _idx = _i + _start; \ - if (_idx >= _arena_cycle) { _idx -= _arena_cycle; } /* adjust so we rotate through the cycle */ \ - } \ - else { \ - _idx = _i; /* remaining arena's after the cycle */ \ - } \ - name_arena = mi_arena_from_index(heap->subproc,_idx); \ + _idx = _i; /* remaining arena's after the cycle */ \ } \ + name_arena = mi_arena_from_index(heap->subproc,_idx); \ + if (req_arena != NULL && name_arena != req_arena && \ + (name_arena == NULL || name_arena->parent != req_arena)) continue; /* only the requested arena or its children */ \ if (name_arena != NULL) \ { diff --git a/test/test-api.c b/test/test-api.c index abbeb434e..018dfa9ac 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -499,6 +499,46 @@ int main(void) { CHECK_BODY("arena_reserve") { result = (0==mi_reserve_os_memory(16*MI_GiB,false,true)); } + + #if TEST_ARENAS // normally disabled as it takes long and consumes a lot of memory + CHECK_BODY("arena_reserve_child") { + // With the following setup, the arena size is 24 GiB and the total + // allocation requires over 16 GiB. mimalloc arena internally creates + // child arenas once the arena size is over 16 GiB so this test will + // trigger the creation and access to child arenas to ensure the arena + // traversal and space accounting works as expected. + const size_t arena_size = 24 * MI_GiB; + const size_t allocation_size = 63 * MI_MiB; + const size_t allocation_count = 272; + const size_t alignment = mi_arena_min_alignment(); + + mi_arena_id_t arena_id = NULL; + bool ok = (0==mi_reserve_os_memory_ex(arena_size,false /* commit */,false /* allow large */, + true /* exclusive */, &arena_id)); + if (!ok) { + fprintf(stderr, "failed to register the 24 GiB arena\n"); + ok = true; // don't fail the test on small machines + } + else { + mi_heap_t* heap = mi_heap_new_in_arena(arena_id); + for (size_t i = 0; i < allocation_count && ok; i++) { + void* p = mi_heap_malloc(heap, allocation_size); + if (p == NULL) { + fprintf(stderr, "allocation %zu failed after %zu MiB; child arena was not used\n", i, i * allocation_size / MI_MiB); + ok = false; + } + if (!mi_arena_contains(arena_id, p)) { + fprintf(stderr, "allocation %zu came from outside the requested arena\n", i); + ok = false; + } + } + fprintf(stderr, "allocated %zu MiB from the parent arena and its children\n", allocation_count * allocation_size / MI_MiB); + mi_heap_destroy(heap); + } + // mi_arena_destroy(arena_id); + result = ok; + } + #endif #endif