diff --git a/ChangeLog.md b/ChangeLog.md index c2e9b8ac929fb..da28fdba2f6a2 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -26,7 +26,7 @@ See docs/process.md for more on how version tagging works. of `clang`. Previously, `.dwo` files were named `.dwo` and other auxiliary files were written to the temporary directory. (#28045) - libunwind and compiler-rt were updated to LLVM 23.1.3. (#28011, #28014) -- mimalloc was updated to 3.5.3. (#28026) +- mimalloc was updated to 3.6.0. (#28026, #28068) 6.0.13 - 10/19/26 ----------------- diff --git a/system/lib/mimalloc/README.emscripten b/system/lib/mimalloc/README.emscripten index 086c612fb57d3..bd9898cc32541 100644 --- a/system/lib/mimalloc/README.emscripten +++ b/system/lib/mimalloc/README.emscripten @@ -1,5 +1,5 @@ -This contains mimalloc d4881d338125e1cb7c47ba4cfb398d6f7c0c8d45 (v3.5.3) with +This contains mimalloc b18ab4eef383f655e760881b02eff8af02800879 (v3.6.0) with Emscripten-specific changes. Origin: https://github.com/microsoft/mimalloc diff --git a/system/lib/mimalloc/include/mimalloc-new-delete.h b/system/lib/mimalloc/include/mimalloc-new-delete.h index bb1bc50750ac1..00aa0b306e374 100644 --- a/system/lib/mimalloc/include/mimalloc-new-delete.h +++ b/system/lib/mimalloc/include/mimalloc-new-delete.h @@ -20,7 +20,7 @@ terms of the MIT license. A copy of the license can be found in the file // --------------------------------------------------------------------------- #if defined(__cplusplus) #include - #include + #include "mimalloc.h" #if defined(_MSC_VER) && defined(_Ret_notnull_) && defined(_Post_writable_byte_size_) // stay consistent with VCRT definitions diff --git a/system/lib/mimalloc/include/mimalloc-override.h b/system/lib/mimalloc/include/mimalloc-override.h index 46e054528bd6e..c34093fe00dde 100644 --- a/system/lib/mimalloc/include/mimalloc-override.h +++ b/system/lib/mimalloc/include/mimalloc-override.h @@ -15,7 +15,7 @@ each source file in a project (but be careful when using external code to not accidentally mix pointers from different allocators). -----------------------------------------------------------------------------*/ -#include +#include "mimalloc.h" // Standard C allocation #define malloc(n) mi_malloc(n) diff --git a/system/lib/mimalloc/include/mimalloc-profile.h b/system/lib/mimalloc/include/mimalloc-profile.h index 5a1327ff70d72..eea3a946afb98 100644 --- a/system/lib/mimalloc/include/mimalloc-profile.h +++ b/system/lib/mimalloc/include/mimalloc-profile.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_PROFILE_H #define MIMALLOC_PROFILE_H -#include +#include "mimalloc.h" #include // bool #include // uint64_t @@ -23,29 +23,33 @@ typedef struct mi_profiler_s mi_profiler_t; // Profiler data is stored together with each sampled allocation (unless the `on_free` field in the profiler is NULL.) typedef struct mi_profiler_sample_data_s { - size_t user_data_size; // size of the user_data (should be the `mi_profiler_t.sample_data_size`) + size_t user_data_size; // size of the user_data (should be the `mi_profiler_t.sample_data_size`) void* user_data[1]; // default, but can be less or more (up to 1KiB), depending on `sample_data_size` } mi_profiler_sample_data_t; // Profiling callback invoked on each sampled allocation. // If `profiler_data!=NULL` (i.e. when `on_free` is not NULL), then `profiler_data->requested_size == requested_size`. -typedef size_t (mi_cdecl mi_profiler_on_alloc_fun )(mi_profiler_t* profiler, mi_profiler_sample_data_t* profiler_data, void* ptr, size_t requested_size, size_t bytes_sample_rate, uint64_t bytes_since_last_sample, const mi_heap_t* heap); +typedef size_t (mi_cdecl mi_profiler_on_alloc_fun)(mi_profiler_t* profiler, mi_profiler_sample_data_t* profiler_data, void* ptr, size_t requested_size, size_t bytes_sample_rate, uint64_t bytes_since_last_sample, mi_heap_t* heap); // Profiling callback invoked on each sampled in-place re-allocation. -typedef size_t (mi_cdecl mi_profiler_on_realloc_inplace_fun)(mi_profiler_t* profiler, mi_profiler_sample_data_t* profiler_data, void* ptr, size_t old_size, const mi_heap_t* heap); +typedef size_t (mi_cdecl mi_profiler_on_realloc_inplace_fun)(mi_profiler_t* profiler, mi_profiler_sample_data_t* profiler_data, void* ptr, size_t old_size, mi_heap_t* heap); // Profiling callback invoked on a previously sampled allocation. -typedef void (mi_cdecl mi_profiler_on_free_fun )(mi_profiler_t* profiler, mi_profiler_sample_data_t* profiler_data, void* ptr, const mi_heap_t* heap); +typedef void (mi_cdecl mi_profiler_on_free_fun)(mi_profiler_t* profiler, mi_profiler_sample_data_t* profiler_data, void* ptr, mi_heap_t* heap); + +// Profiling callback invoked on mi_profiler_snapshot() +typedef void (mi_cdecl mi_profiler_on_snapshot_fun)(mi_profiler_t* profiler); // A profiler // All fields are considered immutable such that they can be copied and accessed concurrently. All fields can be NULL/0. struct mi_profiler_s { - void* reserved; // opaque; reserved by mimalloc - size_t sample_data_size; // size of required profiler data for each sampled allocation (or zero for no data) - size_t initial_sample_rate;// initial sample rate in bytes (set to at least 1 or higher) (can be adjusted by `on_alloc`) - mi_profiler_on_alloc_fun* on_alloc; // called on a sampled allocation (may be called concurrently) - mi_profiler_on_free_fun* on_free; // called on when previous sampled allocation is freed (may be called concurrently) + void* reserved; // opaque; reserved by mimalloc + size_t sample_data_size; // size of required profiler data for each sampled allocation (or zero for no data) + size_t initial_sample_rate;// initial sample rate in bytes (set to at least 1 or higher) (can be adjusted by `on_alloc`) + mi_profiler_on_alloc_fun* on_alloc; // called on a sampled allocation (may be called concurrently) + mi_profiler_on_free_fun* on_free; // called on when previous sampled allocation is freed (may be called concurrently) mi_profiler_on_realloc_inplace_fun* on_realloc_inplace; // (as yet unused) called on in-place reallocation of a previous sampled allocation (may be called concurrently) + mi_profiler_on_snapshot_fun* on_snapshot; // called on mi_profiler_snapshot() (only one thread at a time) // ... more user fields allowed }; @@ -63,7 +67,7 @@ mi_decl_export void mi_heap_profile_disable(mi_heap_t* heap); // attach a profiler to any (current and future) heaps in a sub-process (unless those heaps disabled profiling) mi_decl_export bool mi_subproc_profile(mi_subproc_id_t subproc_id, mi_profiler_t* profiler); -// attach a profiler to any heaps in the main sub-process +// attach a profiler to any heaps in the main sub-process (use NULL to detach) mi_decl_export bool mi_profile(mi_profiler_t* profiler); // start sampling @@ -72,6 +76,20 @@ mi_decl_export bool mi_profiler_start(mi_profiler_t* profiler); // end sampling mi_decl_export bool mi_profiler_stop(mi_profiler_t* profiler); +// snapshot the current profile (may be called concurrently but only one thread will actually call the snapshot callback) +mi_decl_export void mi_profiler_snapshot(mi_profiler_t* profiler); + +// builtin pprof compatible profiler +// the builtin profiler can also be activated using environment variables (see [`pprof.c`](/src/profile/pprof.c)) +// `base_file_name` selects the text snapshot format if it ends in `.heap` or `.text` (the extension is +// stripped); any other extension, or none, uses the default (protobuf) snapshot format. +// `alloc_interval_size`, if >0, automatically snapshots every that many allocated bytes. +// `inuse_interval_size`, if >0, automatically snapshots every time the in-use bytes grow by that many bytes. +// `time_interval_secs`, if >0, automatically snapshots every that many seconds (checked from allocation events). +mi_decl_export mi_profiler_t* mi_pprof_profiler_new(size_t initial_threshold, const char* base_file_name, size_t alloc_interval_size, size_t inuse_interval_size, size_t time_interval_secs); +mi_decl_export void mi_pprof_profiler_delete(mi_profiler_t* profiler); + + #ifdef __cplusplus } #endif diff --git a/system/lib/mimalloc/include/mimalloc-stats.h b/system/lib/mimalloc/include/mimalloc-stats.h index ab45a0715f70b..85198984aed12 100644 --- a/system/lib/mimalloc/include/mimalloc-stats.h +++ b/system/lib/mimalloc/include/mimalloc-stats.h @@ -8,11 +8,11 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_STATS_H #define MIMALLOC_STATS_H -#include +#include "mimalloc.h" #include // memset #include // int64_t -#define MI_STAT_VERSION 5 // increased on every backward incompatible change +#define MI_STAT_VERSION 6 // increased on every backward incompatible change // alignment for atomic fields #if defined(_MSC_VER) @@ -47,8 +47,10 @@ typedef struct mi_stat_counter_s { MI_STAT_COUNT(page_committed) /* committed memory inside pages */ \ MI_STAT_COUNT(pages_abandoned) /* abandoned pages count */ \ MI_STAT_COUNT(threads) /* number of threads */ \ - MI_STAT_COUNT(malloc_normal) /* allocated bytes <= MI_LARGE_OBJ_SIZE_MAX */ \ - MI_STAT_COUNT(malloc_huge) /* allocated bytes in huge pages */ \ + MI_STAT_COUNT(allocated) /* allocated bytes */ \ + MI_STAT_COUNTER(malloc_count) /* total number of malloc calls */ \ + MI_STAT_COUNTER(free_count) /* total number of free calls */ \ + MI_STAT_COUNTER(freed) /* total free'd bytes (note: total allocated is `allocated.total`) */ \ MI_STAT_COUNTER(malloc_requested) /* malloc requested bytes */ \ \ MI_STAT_COUNTER(mmap_calls) \ @@ -56,8 +58,6 @@ typedef struct mi_stat_counter_s { MI_STAT_COUNTER(reset_calls) \ MI_STAT_COUNTER(purge_calls) \ MI_STAT_COUNTER(arena_count) /* number of memory arena's */ \ - MI_STAT_COUNTER(malloc_normal_count) /* number of blocks <= MI_LARGE_OBJ_SIZE_MAX */ \ - MI_STAT_COUNTER(malloc_huge_count) /* number of huge bloks */ \ MI_STAT_COUNTER(malloc_guarded_count) /* number of allocations with guard pages */ \ \ /* internal statistics */ \ @@ -81,10 +81,17 @@ typedef struct mi_stat_counter_s { MI_STAT_COUNTER(pages_reclaim_on_free) \ MI_STAT_COUNTER(pages_reabandon_full) \ MI_STAT_COUNTER(pages_unabandon_busy_wait) \ + MI_STAT_COUNTER(pages_unown_cas_retry) \ MI_STAT_COUNTER(heaps_delete_wait) \ MI_STAT_COUNTER(pages_stat_updates) /* calls to successful page_stat_update */ \ MI_STAT_COUNTER(pages_stat_update_count) /* total free/allocs */ \ MI_STAT_COUNTER(profile_samples) /* total sampled profiled allocations */ \ + /* deprecated */ \ + MI_STAT_COUNT(malloc_normal) /* allocated bytes <= MI_LARGE_OBJ_SIZE_MAX */ \ + MI_STAT_COUNT(malloc_huge) /* allocated bytes in huge pages */ \ + MI_STAT_COUNTER(malloc_normal_count) /* number of blocks <= MI_LARGE_OBJ_SIZE_MAX */ \ + MI_STAT_COUNTER(malloc_huge_count) /* number of huge bloks */ \ + // Size bins for chunks typedef enum mi_chunkbin_e { @@ -105,8 +112,8 @@ typedef enum mi_chunkbin_e { typedef struct mi_stats_s { - size_t size; // size of the mi_stats_t structure - size_t version; + size_t size; // size of the mi_stats_t structure + size_t version; mi_decl_align(8) MI_STAT_FIELDS() @@ -133,6 +140,8 @@ static inline void mi_stats_init(mi_stats_t* stats) { mi_stats_header_init(stats); } +// Always use this to declare and initialize a `mi_stats_t` variable such +// that the version and size can be checked at runtime. #define mi_stats_t_decl(name) mi_stats_t name; mi_stats_init(&name); // Exported definitions diff --git a/system/lib/mimalloc/include/mimalloc.h b/system/lib/mimalloc/include/mimalloc.h index 2985edd9492a5..df3b0ce41a6e6 100644 --- a/system/lib/mimalloc/include/mimalloc.h +++ b/system/lib/mimalloc/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 30503 // major + 2 digits minor + 2 digits patch +#define MI_MALLOC_VERSION 30600 // major + 2 digits minor + 2 digits patch // ------------------------------------------------------ // Compiler specific attributes @@ -139,11 +139,14 @@ mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept; // `mi_free_small` is for special applications like language runtimes. // it should only be used to free objects from `mi_*alloc_small` and is potentially a tiny bit faster than `mi_free` -mi_decl_export void mi_free_small(void* p) mi_attr_noexcept; +mi_decl_export void mi_free_small(void* p) mi_attr_noexcept; // As `mi_free_small` but `p` should not be a NULL pointer. mi_decl_export void mi_free_small_nonnull(void* p) mi_attr_noexcept; +// For runtime's: return the current owner thread-id of the page containing an allocation. +mi_decl_nodiscard mi_decl_export size_t mi_thread_id_of(const void* p) mi_attr_noexcept; + // ------------------------------------------------------------------------------------- // Aligned allocation @@ -240,9 +243,11 @@ struct mi_heap_s; typedef struct mi_heap_s mi_heap_t; mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new(void); + mi_decl_export void mi_heap_delete(mi_heap_t* heap); // move live blocks to the main heap mi_decl_export void mi_heap_destroy(mi_heap_t* heap); // free all live blocks mi_decl_export void mi_heap_set_numa_affinity(mi_heap_t* heap, int numa_node); +mi_decl_export void mi_heap_set_debug_fill_preserve(mi_heap_t* heap, size_t preserve); // in debug mode, do not fill the first `preserve` bytes of blocks on allocation or free mi_decl_export void mi_heap_collect(mi_heap_t* heap, bool force); mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_main(void); @@ -317,6 +322,7 @@ typedef struct mi_heap_area_s { typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg); +// Heap visitors assume no concurrent heap access (except thread exit). mi_decl_export bool mi_heap_visit_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg); mi_decl_export bool mi_heap_visit_abandoned_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg); @@ -336,6 +342,7 @@ mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_comm mi_decl_export void mi_debug_show_arenas(void) mi_attr_noexcept; mi_decl_export void mi_arenas_print(void) mi_attr_noexcept; +mi_decl_export void mi_arenas_purge(void) mi_attr_noexcept; mi_decl_export size_t mi_arena_min_alignment(void); mi_decl_export size_t mi_arena_min_size(void); mi_decl_export size_t mi_arena_max_object_size(void); @@ -421,6 +428,13 @@ mi_decl_export bool mi_manage_memory(void* start, size_t size, bool is_committe // unsafe: assumes the page belonging to `p` is only accessed by the calling thread. mi_decl_export bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc_threshold) mi_attr_noexcept; +// Owner-thread only: abandon pages and delete the theap (not its heap or live blocks). +mi_decl_export void mi_theap_delete(mi_theap_t* theap) mi_attr_noexcept; + +// Abandon all theap pages, preserving live blocks and the theap for reuse. The owner must be the current thread or quiescent outside mimalloc (excluding concurrent heap/theap deletion). +mi_decl_export void mi_theap_quiescent_abandon(mi_theap_t* theap) mi_attr_noexcept; + + // ------------------------------------------------------ // Deprecated // ------------------------------------------------------ @@ -451,7 +465,7 @@ typedef enum mi_option_e { mi_option_show_stats, // print statistics on termination mi_option_verbose, // print verbose messages // advanced options - mi_option_deprecated_eager_commit, + mi_option_deprecated_eager_commit, mi_option_arena_eager_commit, // eager commit arenas? Use 2 to enable just on overcommit systems (=2) mi_option_purge_decommits, // should a memory purge decommit? (=1). Set to 0 to use memory reset on a purge (instead of decommit) mi_option_allow_large_os_pages, // allow use of large (2 or 4 MiB) OS pages, implies eager commit. @@ -462,7 +476,7 @@ typedef enum mi_option_e { mi_option_deprecated_page_reset, mi_option_deprecated_abandoned_page_purge, mi_option_deprecated_segment_reset, - mi_option_deprecated_eager_commit_delay, + mi_option_deprecated_eager_commit_delay, mi_option_purge_delay, // memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all. (=10) mi_option_use_numa_nodes, // 0 = use all available numa nodes, otherwise use at most N nodes. mi_option_disallow_os_alloc, // 1 = do not use OS memory for allocation (but only programmatically reserved arenas) @@ -479,11 +493,11 @@ typedef enum mi_option_e { mi_option_deprecated_visit_abandoned, // allow visiting theap blocks from abandoned threads (=0) mi_option_guarded_min, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects (=0) mi_option_guarded_max, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects (=0) - mi_option_guarded_precise, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0) - mi_option_guarded_sample_rate, // 1 out of N allocations in the min/max range will be guarded (=1000) + mi_option_unsafe_guarded_precise, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0) + mi_option_guarded_sample_rate, // guard an object in the min/max range about every N requested bytes (=0) mi_option_guarded_sample_seed, // can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0) mi_option_generic_collect, // collect theaps every N (=10000) generic allocation calls - mi_option_page_reclaim_on_free, // reclaim abandoned pages on a free (=0). -1 disallow always, 0 allows if the page originated from the current theap, 1 allow always + mi_option_page_reclaim_on_free, // heap-creation default for reclaim on free: -1 disallow, 0 originating theap only, 1 allow across theaps mi_option_page_full_retain, // retain N full (small) pages per size class (=2) mi_option_page_max_candidates, // max candidate pages to consider for allocation (=4) mi_option_max_vabits, // max user space virtual address bits to consider (=48) @@ -492,17 +506,24 @@ typedef enum mi_option_e { mi_option_page_max_reclaim, // don't reclaim pages of the same originating theap if we already own N pages (in that size class) (=-1 (unlimited)) mi_option_page_cross_thread_max_reclaim, // don't reclaim pages across threads if we already own N pages (in that size class) (=32) mi_option_allow_thp, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process. - mi_option_minimal_purge_size, // set minimal purge size (in KiB) (=0). By default set to either 64 or 2048 if THP is enabled. - mi_option_arena_max_object_size, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit). + mi_option_arena_purge_min_size, // set minimal purge size (in KiB) (=0). By default set to either 64 or 2048 if THP is enabled. + mi_option_arena_max_object_size, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit). mi_option_arena_is_numa_local, // experimental: associate local numa node with an initial arena allocation - mi_option_collect_merges_stats, // on each theap collection, the theap stats are merged automatically with the parent heap + mi_option_stats_merge_on_collect, // on each theap collection, the theap stats are merged automatically with the parent heap + mi_option_profile_alloc_interval, // number of allocated bytes between automatic snapshots of the heap profiler (=1GiB) + mi_option_profile_inuse_interval, // number of in-use bytes grown between automatic snapshots of the heap profiler (=0, disabled) + mi_option_profile_time_interval, // number of seconds between automatic snapshots of the heap profiler (=0, disabled) + mi_option_profile_sample_rate, // sample rate (in bytes) for heap profiler (=512 KiB) + mi_option_stats_merge_threshold, // on a page stats update, merge theap stats into the parent heap if N bytes were freed/allocated; 0 disables (=512 KiB) + mi_option_arena_purge_immediate_size, // ranges above this size are purged immediately _mi_option_last, // legacy option names mi_option_large_os_pages = mi_option_allow_large_os_pages, mi_option_eager_region_commit = mi_option_arena_eager_commit, mi_option_reset_decommits = mi_option_purge_decommits, mi_option_reset_delay = mi_option_purge_delay, - mi_option_limit_os_alloc = mi_option_disallow_os_alloc + mi_option_limit_os_alloc = mi_option_disallow_os_alloc, + mi_option_minimal_purge_size = mi_option_arena_purge_min_size } mi_option_t; @@ -553,8 +574,8 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const // -------------------------------------------------------- // C++ wrappers -// The `mi_new` wrappers implement C++ semantics on out-of-memory -// instead of directly returning `NULL`. (and call `std::get_new_handler` +// The `mi_new` wrappers implement C++ semantics on out-of-memory +// instead of directly returning `NULL`. (and call `std::get_new_handler` // and potentially raise a `std::bad_alloc` exception). // -------------------------------------------------------- @@ -576,9 +597,9 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_alloc_new_nothr // -------------------------------------------------------------------------- // Inlined constant size allocations. -// These are meant for runtime systems, or overrides where we need the best +// These are meant for runtime systems, or overrides where we need the best // performance for small, constant-size allocations. -// These are only better than the regular functions if the size or alignment +// These are only better than the regular functions if the size or alignment // are indeed constant at the call site. // -------------------------------------------------------------------------- @@ -598,7 +619,7 @@ static inline mi_decl_restrict void* mi_malloc_csize(size_t size) mi_attr_noexce static inline mi_decl_restrict void* mi_zalloc_csize(size_t size) mi_attr_noexcept { if (size <= MI_SMALL_SIZE_MAX) { return mi_wzalloc_small(mi_wsize_from_size(size)); } else { return mi_zalloc(size); } } -static inline mi_decl_restrict void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { +static inline mi_decl_restrict void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { assert(theap!=NULL); if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_wmalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_malloc(theap,size); } } @@ -622,6 +643,13 @@ static inline void mi_free_csize_aligned_nonnull(void* p, size_t size, size_t al if (aligned <= size && size <= MI_SMALL_SIZE_MAX) { mi_free_small_nonnull(p); } else { mi_free(p); } } +// As `mi_free_small` but `p` must be allocated on a page owned by the current thread (be very careful!) +mi_decl_export void mi_free_small_local(void* p) mi_attr_noexcept; + +// As `mi_free_small_local` but `p` cannot be NULL as well. +mi_decl_export void mi_free_small_local_nonnull(void* p) mi_attr_noexcept; + + // ------------------------------------------------------ // C++ standard library allocator interface. // ------------------------------------------------------ diff --git a/system/lib/mimalloc/include/mimalloc/alloc/alloc.h b/system/lib/mimalloc/include/mimalloc/alloc/alloc.h new file mode 100644 index 0000000000000..314c5150dc8b2 --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/alloc/alloc.h @@ -0,0 +1,37 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_INTERNAL_ALLOC_H +#define MI_INTERNAL_ALLOC_H + +#include "../types.h" +#include "../util/libc.h" +#include "../util/options.h" + +mi_decl_restrict void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` +mi_decl_restrict void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` +void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero) mi_attr_noexcept; + +// Safe multiply `count*size` into `total`; return `true` on overflow. +static inline bool mi_count_size_overflow(size_t count, size_t size, size_t* total) { + if (count==1) { // quick check for the case where count is one (common for C++ allocators) + *total = size; + return false; + } + else if mi_likely(!mi_mul_overflow(count, size, total)) { + return false; + } + else { + #if MI_DEBUG > 0 + _mi_error_message(EOVERFLOW, "allocation request is too large (%zu * %zu bytes)\n", count, size); + #endif + *total = SIZE_MAX; + return true; + } +} + +#endif diff --git a/system/lib/mimalloc/include/mimalloc/alloc/free.h b/system/lib/mimalloc/include/mimalloc/alloc/free.h new file mode 100644 index 0000000000000..2f06ea861ee65 --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/alloc/free.h @@ -0,0 +1,18 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_INTERNAL_FREE_H +#define MI_INTERNAL_FREE_H + +#include "../types.h" + +mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p); +void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size); +void _mi_free_subproc_safe(void* p) mi_attr_noexcept; +size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexcept; + +#endif diff --git a/system/lib/mimalloc/include/mimalloc/alloc/page.h b/system/lib/mimalloc/include/mimalloc/alloc/page.h new file mode 100644 index 0000000000000..acbda3bc6d710 --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/alloc/page.h @@ -0,0 +1,605 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_INTERNAL_PAGE_H +#define MI_INTERNAL_PAGE_H + +#include "../types.h" +#include "../prim/os.h" +#include "../heap/heap.h" +#include "../heap/page-map.h" +#include "../track.h" + +mi_decl_restrict void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; +mi_decl_restrict void* _mi_malloc_generic_no_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; + +void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks +void _mi_page_unfull(mi_page_t* page); +bool _mi_page_free(mi_page_t* page, mi_page_queue_t* pq); // false if page release is delayed +void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread... +void _mi_deferred_free(mi_theap_t* theap, bool force); +bool _mi_page_free_collect(mi_page_t* page, bool force); // returns `true` if cross-thread free'd blocks were collected +mi_block_t* _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head); +mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page); +void _mi_page_update_stats(mi_page_t* page); + +void _mi_theap_collect_retired(mi_theap_t* theap, bool force); +bool _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page); // true if an empty page was freed +#if MI_DEBUG>1 +bool _mi_page_is_valid(mi_page_t* page); +#endif + +// in page-queue.c +bool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq); +size_t _mi_page_stats_bin(const mi_page_t* page); // for stats +size_t _mi_bin_size(size_t bin); // for stats +size_t _mi_bin(size_t size); // for stats + + +static inline mi_page_queue_t* mi_page_queue(const mi_theap_t* theap, size_t size) { + mi_page_queue_t* const pq = &((mi_theap_t*)theap)->pages[_mi_bin(size)]; + if (size <= MI_LARGE_MAX_OBJ_SIZE) { mi_assert_internal(pq->block_size <= MI_LARGE_MAX_OBJ_SIZE); } + return pq; +} + +// Get the block size of a page +static inline size_t mi_page_block_size(const mi_page_t* page) { + mi_assert_internal(page->block_size > 0); + return page->block_size; +} + +// Page start +static inline uint8_t* mi_page_start(const mi_page_t* page) { + // multiplication must be done in `size_t` + return (uint8_t*)page + page->page_offset; +} + +static inline size_t mi_page_size(const mi_page_t* page) { + return mi_page_block_size(page) * page->reserved; +} + +static inline uint8_t* mi_page_area(const mi_page_t* page, size_t* size) { + if (size) { *size = mi_page_size(page); } + return mi_page_start(page); +} + +static inline size_t mi_page_info_size(void) { + return mi_align_up(sizeof(mi_page_t), MI_MAX_ALIGN_SIZE); +} + +// Can the first block in a page with blocks of size `bsize` have a cache offset? (see `MI_PAGE_CACHE_OFFSET`) +static inline bool mi_block_size_has_cache_offset(size_t bsize) { + return (MI_PAGE_CACHE_OFFSET != 0 && bsize > MI_SMALL_MAX_OBJ_SIZE); +} + +// Is a (non-small) block of size `bsize` allocated in a medium page? +// Blocks of 1/8th of a medium page (64 KiB) leave no space for a cache offset and use a large page instead. +static inline bool mi_block_size_is_medium(size_t bsize) { + #if MI_ENABLE_LARGE_PAGES + if (mi_block_size_has_cache_offset(bsize) && bsize == MI_MEDIUM_PAGE_SIZE/8) return false; + #endif + return (bsize <= MI_MEDIUM_MAX_OBJ_SIZE); +} + +// The alignment that every block of size `bsize` is guaranteed to have in any page (or 0 if `bsize==0`). +// This is guaranteed by the block start (see `arena/arena-page.c:mi_page_meta_in_front_block_start` and `mi_page_meta_select`), +// and used by `mi_malloc_aligned` to avoid over-allocation for naturally aligned blocks. +static inline size_t mi_natural_alignment_of_size(size_t bsize) { + const size_t lowbit = bsize & (~bsize + 1); // largest power of 2 that divides `bsize` + if (mi_is_power_of_two(bsize) && bsize <= MI_PAGE_MAX_START_BLOCK_ALIGN2) return bsize; // power-of-two blocks up to 4KiB + if (bsize <= MI_SMALL_SIZE_MAX) return lowbit; // small blocks start at a multiple of their size + if ((bsize % MI_PAGE_OSPAGE_BLOCK_ALIGN2) == 0) { // multiples of 4KiB are 4KiB aligned (or cache offset unit aligned) + return (mi_block_size_has_cache_offset(bsize) ? MI_PAGE_CACHE_OFFSET_UNIT : MI_PAGE_OSPAGE_BLOCK_ALIGN2); + } + return (lowbit < MI_MAX_ALIGN_SIZE ? lowbit : MI_MAX_ALIGN_SIZE); // otherwise blocks start at least `MI_MAX_ALIGN_SIZE` aligned +} + +static inline bool mi_page_contains_address(const mi_page_t* page, const void* p) { + size_t psize; + uint8_t* start = mi_page_area(page, &psize); + return (start <= (uint8_t*)p && (uint8_t*)p < start + psize); +} + +static inline bool mi_page_is_in_arena(const mi_page_t* page) { + return (page->memid.memkind == MI_MEM_ARENA); +} + +static inline bool mi_page_is_singleton(const mi_page_t* page) { + return (page->reserved == 1); +} + +// Get the usable block size of a page without fixed padding. +// This may still include internal padding due to alignment and rounding up size classes. +static inline size_t mi_page_usable_block_size(const mi_page_t* page) { + return mi_page_block_size(page) - MI_PADDING_SIZE; +} + +static inline bool mi_page_meta_is_separated(const mi_page_t* page) { + #if MI_PAGE_META_IS_ALIGNED + #if MI_PAGE_META_SMALL_IS_ALIGNED + return (page != mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); + #else + MI_UNUSED_RELEASE(page); + mi_assert_internal(page != mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); + return true; + #endif + #elif MI_PAGE_META_IS_SEPARATED + // usually separated but can still be in front for direct OS allocations (due to size or alignment) or due to MI_PAGE_META_SMALL_IS_ALIGNED + return (page->memid.memkind == MI_MEM_ARENA && page != mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); + #else + MI_UNUSED(page); + return false; + #endif +} + +static inline uint8_t* mi_page_slice_start(const mi_page_t* page) { + if (mi_page_meta_is_separated(page)) { + // page meta info is at a separate location (at `arena->pages`) + return (uint8_t*)mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN); + } + else { + // page meta info is at the start of the page slices + return (uint8_t*)page; + } +} + +// This gives the offset relative to the start slice of a page. +static inline size_t mi_page_slice_offset_of(const mi_page_t* page, size_t offset_relative_to_page_start) { + return (mi_page_start(page) - mi_page_slice_start(page)) + offset_relative_to_page_start; +} + +// How much of the page is committed relative to the slice start? (or 0 if fully committed already) +static inline size_t mi_page_slice_committed(const mi_page_t* page) { + return ((size_t)page->slice_pcommitted * _mi_os_page_size()); +} + +// Currently committed part of a page +static inline size_t mi_page_committed(const mi_page_t* page) { + const size_t slice_committed = mi_page_slice_committed(page); + return (slice_committed == 0 ? mi_page_size(page) : slice_committed - mi_page_slice_offset_of(page,0)); +} + +static inline size_t mi_page_used(const mi_page_t* page) { + mi_assert_internal(page != NULL); + return mi_xused_used_count(page->xused); +} + +static inline void mi_page_used_reset(mi_page_t* page) { + page->xused = mi_xused_used_reset(page->xused); +} + +static inline size_t mi_page_alloc_count(const mi_page_t* page) { + return mi_xused_alloc_count(page->xused); +} + +static inline size_t mi_page_last_used(const mi_page_t* page) { + #if MI_SIZE_SIZE >= 8 + return (page->xused.used_alloc >> 32) & 0xFFFF; + #else + return page->xlast_used; + #endif +} + +static inline size_t mi_page_last_alloc(const mi_page_t* page) { + #if MI_SIZE_SIZE >= 8 + return (page->xused.used_alloc >> 48) & 0xFFFF; + #else + return page->xlast_alloc; + #endif +} + + +// are all blocks in a page freed? +// note: needs up-to-date used count, (as the `xthread_free` list may not be empty). see `_mi_page_collect_free`. +static inline bool mi_page_all_free(const mi_page_t* page) { + mi_assert_internal(page != NULL); + return (mi_page_used(page)==0); +} + +// are there immediately available blocks, i.e. blocks available on the free list. +static inline bool mi_page_immediate_available(const mi_page_t* page) { + mi_assert_internal(page != NULL); + return (page->free != NULL); +} + + +// is the page not yet used up to its reserved space? +static inline bool mi_page_is_expandable(const mi_page_t* page) { + mi_assert_internal(page != NULL); + mi_assert_internal(page->capacity <= page->reserved); + return (page->capacity < page->reserved); +} + + +static inline bool mi_page_is_full(const mi_page_t* page) { + const bool full = (page->reserved == mi_page_used(page)); + mi_assert_internal(!full || page->free == NULL); + return full; +} + +// is more than (1 - 1/n)'th of a page in use? +static inline bool mi_page_is_used_at_frac(const mi_page_t* page, size_t n) { + if (page==NULL) return true; + const size_t frac = page->reserved / n; + return (page->reserved - mi_page_used(page) <= frac); +} + +// is more than 7/8th of a page in use? +static inline bool mi_page_is_mostly_used(const mi_page_t* page) { + return mi_page_is_used_at_frac(page, 8); +} + +static inline bool mi_page_is_small(const mi_page_t* page) { + return (page->block_size <= MI_SMALL_MAX_OBJ_SIZE && !mi_page_is_singleton(page)); +} + +static inline bool mi_page_is_huge(const mi_page_t* page) { + return (mi_page_is_singleton(page) && + (page->block_size > MI_LARGE_MAX_OBJ_SIZE || + (mi_memkind_is_os(page->memid.memkind) && page->memid.mem.os.base < (void*)page))); +} + +static inline size_t mi_page_min_commit_size(void) { + const size_t psize = _mi_os_page_size(); + return (MI_PAGE_MIN_COMMIT_SIZE >= psize ? MI_PAGE_MIN_COMMIT_SIZE : psize); +} + +//----------------------------------------------------------- +// Page thread id and flags +//----------------------------------------------------------- + +// Thread id of thread that owns this page (with flags in the bottom 2 bits) +static inline mi_threadid_t mi_page_xthread_id(const mi_page_t* page) { + return mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_id); +} + +// Plain thread id of the thread that owns this page +static inline mi_threadid_t mi_page_thread_id(const mi_page_t* page) { + return (mi_page_xthread_id(page) & ~MI_PAGE_FLAG_MASK); +} + +static inline mi_page_flags_t mi_page_flags(const mi_page_t* page) { + return (mi_page_xthread_id(page) & MI_PAGE_FLAG_MASK); +} + +static inline bool mi_page_flags_set(mi_page_t* page, bool set, mi_page_flags_t newflag) { + mi_page_flags_t old; + if (set) { old = mi_atomic_or_relaxed(&page->xthread_id, newflag); } + else { old = mi_atomic_and_relaxed(&page->xthread_id, ~newflag); } + return ((old & newflag) == newflag); +} + +static inline bool mi_page_is_in_full(const mi_page_t* page) { + return ((mi_page_flags(page) & MI_PAGE_IN_FULL_QUEUE) != 0); +} + +static inline void mi_page_set_in_full(mi_page_t* page, bool in_full) { + const bool was_in_full = mi_page_flags_set(page, in_full, MI_PAGE_IN_FULL_QUEUE); + if (was_in_full != in_full) { + // optimize: maintain pages_full_size to avoid visiting the full queue (issue #1220) + mi_theap_t* const theap = page->theap; + mi_assert_internal(theap!=NULL); + if (theap != NULL) { + mi_assert_internal(page->capacity==page->reserved); + const size_t size = page->reserved * mi_page_block_size(page); + if (in_full) { theap->pages_full_size += size; } + else { mi_assert_internal(size <= theap->pages_full_size); theap->pages_full_size -= size; } + } + } +} + +static inline bool mi_page_has_interior_pointers(const mi_page_t* page) { + return ((mi_page_flags(page) & MI_PAGE_HAS_INTERIOR_POINTERS) != 0); +} + +static inline void mi_page_set_has_interior_pointers(mi_page_t* page, bool has_aligned) { + mi_page_flags_set(page, has_aligned, MI_PAGE_HAS_INTERIOR_POINTERS); +} + +static inline void mi_page_set_theap(mi_page_t* page, mi_theap_t* theap) { + // mi_assert_internal(!mi_page_is_in_full(page)); // can happen when destroying pages on theap_destroy + page->theap = theap; + const mi_threadid_t tid = (theap == NULL ? MI_THREADID_ABANDONED : theap->tld->thread_id); + mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0); + + // we need to use an atomic cas since a concurrent thread may still set the MI_PAGE_HAS_INTERIOR_POINTERS flag (see `alloc_aligned.c`). + mi_threadid_t xtid_old = mi_page_xthread_id(page); + mi_threadid_t xtid; + do { + xtid = tid | (xtid_old & MI_PAGE_FLAG_MASK); + } while (!mi_atomic_cas_weak_release(&page->xthread_id, &xtid_old, xtid)); +} + +static inline bool mi_page_is_abandoned(const mi_page_t* page) { + // note: the xtheap field of an abandoned theap is set to the subproc (for fast reclaim-on-free) + return (mi_page_thread_id(page) <= MI_THREADID_ABANDONED_MAPPED); +} + +static inline bool mi_page_is_abandoned_mapped(const mi_page_t* page) { + return (mi_page_thread_id(page) == MI_THREADID_ABANDONED_MAPPED); +} + +static inline void mi_page_set_abandoned_mapped(mi_page_t* page) { + mi_assert_internal(mi_page_is_abandoned(page)); + mi_atomic_or_relaxed(&page->xthread_id, (mi_threadid_t)MI_THREADID_ABANDONED_MAPPED); +} + +static inline void mi_page_clear_abandoned_mapped(mi_page_t* page) { + mi_assert_internal(mi_page_is_abandoned_mapped(page)); + mi_atomic_and_relaxed(&page->xthread_id, (mi_threadid_t)MI_PAGE_FLAG_MASK); +} + + +static inline mi_theap_t* mi_page_theap(const mi_page_t* page) { + mi_assert_internal(!mi_page_is_abandoned(page)); + mi_assert_internal(page->theap != NULL && !_mi_is_empty_theap(page->theap)); + return page->theap; +} + +static inline mi_tld_t* mi_page_tld(const mi_page_t* page) { + mi_assert_internal(!mi_page_is_abandoned(page)); + mi_assert_internal(page->theap != NULL); + return page->theap->tld; +} + + +static inline mi_heap_t* mi_page_heap(const mi_page_t* page) { + mi_heap_t* heap = page->heap; + mi_assert_internal(heap != NULL); + return heap; +} + +static inline mi_subproc_t* mi_page_subproc(const mi_page_t* page) { + mi_heap_t* const heap = mi_page_heap(page); + return heap->subproc; +} + +//----------------------------------------------------------- +// Thread free list and ownership +//----------------------------------------------------------- + +// Thread free flag helpers +static inline mi_block_t* mi_tf_block(mi_thread_free_t tf) { + return (mi_block_t*)(tf & ~1); +} + +static inline bool mi_tf_is_owned(mi_thread_free_t tf) { + return ((tf & 1) == 1); +} + +static inline mi_thread_free_t mi_tf_create(mi_block_t* block, bool owned) { + const uintptr_t base = (uintptr_t)block | (owned ? 1 : 0); + return (mi_thread_free_t)base; +} + +static inline mi_thread_free_t mi_tf_set_owned(mi_thread_free_t tf, bool owned) { + return mi_tf_create(mi_tf_block(tf), owned); +} + +// Thread free access +static inline mi_block_t* mi_page_thread_free(const mi_page_t* page) { + return mi_tf_block(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free)); +} + +// are there any available blocks? +static inline bool mi_page_has_any_available(const mi_page_t* page) { + mi_assert_internal(page != NULL && page->reserved > 0); + return (mi_page_used(page) < page->reserved || (mi_page_thread_free(page) != NULL)); +} + +// Owned? +static inline bool mi_page_is_owned(const mi_page_t* page) { + return mi_tf_is_owned(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free)); +} + +// get ownership; returns true if the page was not owned before. +static inline bool mi_page_claim_ownership(mi_page_t* page) { + const uintptr_t old = mi_atomic_or_acq_rel(&page->xthread_free, (uintptr_t)1); + return ((old&1)==0); +} + +/* ----------------------------------------------------------- + Delayed page release +----------------------------------------------------------- */ + +static inline void mi_page_qsbr_unlink(mi_page_t* page) { +#if MI_QSBR + if (!page->qsbr_in_queue) { + mi_assert_internal(page->qsbr_next == NULL && page->qsbr_prev == NULL); + return; + } + mi_theap_t* theap = mi_page_theap(page); + if (page->qsbr_prev != NULL) { page->qsbr_prev->qsbr_next = page->qsbr_next; } + else { + mi_assert_internal(theap->qsbr_first == page); + theap->qsbr_first = page->qsbr_next; + } + if (page->qsbr_next != NULL) { page->qsbr_next->qsbr_prev = page->qsbr_prev; } + else { + mi_assert_internal(theap->qsbr_last == page); + theap->qsbr_last = page->qsbr_prev; + } + page->qsbr_next = page->qsbr_prev = NULL; + page->qsbr_in_queue = false; +#else + MI_UNUSED(page); +#endif +} + +static inline void mi_page_qsbr_enqueue(mi_page_t* page, mi_theap_t* theap) { +#if MI_QSBR + mi_assert_internal(mi_page_theap(page) == theap); + mi_assert_internal(mi_page_all_free(page)); + mi_assert_internal(mi_heap_qsbr_is_enabled(mi_page_heap(page))); + mi_assert_internal(!mi_page_qsbr_is_clear(&page->qsbr)); + if (page->qsbr_in_queue) return; + mi_assert_internal(page->qsbr_next == NULL && page->qsbr_prev == NULL); + page->qsbr_prev = theap->qsbr_last; + if (theap->qsbr_last != NULL) { theap->qsbr_last->qsbr_next = page; } + else { theap->qsbr_first = page; } + theap->qsbr_last = page; + page->qsbr_in_queue = true; +#else + MI_UNUSED(page); + MI_UNUSED(theap); +#endif +} + +static inline void _mi_page_qsbr_clear(mi_page_t* page) { +#if MI_QSBR + mi_page_qsbr_unlink(page); + mi_page_qsbr_clear(&page->qsbr); +#else + MI_UNUSED(page); +#endif +} + +static inline bool _mi_page_qsbr_is_clear(const mi_page_t* page) { +#if MI_QSBR + return (!page->qsbr_in_queue && page->qsbr_next == NULL && page->qsbr_prev == NULL && + mi_page_qsbr_is_clear(&page->qsbr)); +#else + MI_UNUSED(page); + return true; +#endif +} + +static inline bool _mi_page_qsbr_should_delay(mi_page_t* page) { +#if MI_QSBR + mi_assert_internal(mi_page_is_owned(page) && mi_page_all_free(page)); + mi_heap_t* heap = mi_page_heap(page); + if (!mi_heap_qsbr_is_enabled(heap)) return false; + const bool delayed = mi_page_qsbr_should_delay(&page->qsbr, &heap->qsbr, page->capacity * mi_page_block_size(page)); + mi_assert_internal(!delayed || !mi_page_qsbr_is_clear(&page->qsbr)); + return delayed; +#else + MI_UNUSED(page); + return false; +#endif +} + +/* ------------------------------------------------------------------- +Encoding/Decoding the free list next pointers + +This is to protect against buffer overflow exploits where the +free list is mutated. Many hardened allocators xor the next pointer `p` +with a secret key `k1`, as `p^k1`. This prevents overwriting with known +values but might be still too weak: if the attacker can guess +the pointer `p` this can reveal `k1` (since `p^k1^p == k1`). +Moreover, if multiple blocks can be read as well, the attacker can +xor both as `(p1^k1) ^ (p2^k1) == p1^p2` which may reveal a lot +about the pointers (and subsequently `k1`). + +Instead mimalloc uses an extra key `k2` and encodes as `((p^k2)<<next, keys); + #else + MI_UNUSED(keys); MI_UNUSED(null); + next = (mi_block_t*)block->next; + #endif + mi_track_mem_noaccess(block,sizeof(mi_block_t)); + return next; +} + +static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_block_t* next, const uintptr_t* keys) { + mi_track_mem_undefined(block,sizeof(mi_block_t)); + #if MI_ENCODE_FREELIST + block->next = mi_ptr_encode(null, next, keys); + #else + MI_UNUSED(keys); MI_UNUSED(null); + block->next = (mi_encoded_t)next; + #endif + mi_track_mem_noaccess(block,sizeof(mi_block_t)); +} + +static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) { + #if MI_ENCODE_FREELIST + mi_block_t* next = mi_block_nextx(page,block,page->keys); + // check for free list corruption: is `next` at least in the same page? + // todo: check if `next` is `page->block_size` aligned? + if mi_unlikely(next!=NULL && !mi_page_contains_address(page,next)) { + return _mi_block_next_is_corrupted(page,block,next); // returns NULL + } + return next; + #else + MI_UNUSED(page); + return mi_block_nextx(page,block,NULL); + #endif +} + +static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, const mi_block_t* next) { + #if MI_ENCODE_FREELIST + mi_block_set_nextx(page,block,next, page->keys); + #else + MI_UNUSED(page); + mi_block_set_nextx(page,block,next,NULL); + #endif +} + +#endif diff --git a/system/lib/mimalloc/include/mimalloc/arena/arena-alloc.h b/system/lib/mimalloc/include/mimalloc/arena/arena-alloc.h new file mode 100644 index 0000000000000..710f29cc902a1 --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/arena/arena-alloc.h @@ -0,0 +1,18 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_INTERNAL_ARENA_ALLOC_H +#define MI_INTERNAL_ARENA_ALLOC_H + +#include "../types.h" + +void* _mi_arenas_alloc(mi_heap_t* heap, size_t size, bool commit, bool allow_pinned, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid); +void* _mi_arenas_alloc_aligned(mi_heap_t* heap, size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_pinned, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid); +void _mi_arenas_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid); +void _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld); + +#endif diff --git a/system/lib/mimalloc/include/mimalloc/arena/arena-page.h b/system/lib/mimalloc/include/mimalloc/arena/arena-page.h new file mode 100644 index 0000000000000..40715566b4653 --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/arena/arena-page.h @@ -0,0 +1,25 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_INTERNAL_ARENA_PAGE_H +#define MI_INTERNAL_ARENA_PAGE_H + +#include "../types.h" + +mi_page_t* _mi_arenas_page_alloc(mi_theap_t* theap, size_t block_size, size_t page_alignment); +void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx /* can be NULL */); +void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap); +void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx /* can be NULL */); +bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page); +void _mi_heap_move_pages(mi_heap_t* heap_from, mi_heap_t* heap_to); +void _mi_heap_destroy_pages(mi_heap_t* heap_from); +#if MI_QSBR +// The caller must stop all allocation, freeing, and theap lifecycle operations on this heap. +void _mi_heap_collect_abandoned(mi_heap_t* heap); +#endif + +#endif diff --git a/system/lib/mimalloc/include/mimalloc/arena/arena.h b/system/lib/mimalloc/include/mimalloc/arena/arena.h new file mode 100644 index 0000000000000..18d486a2eb42d --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/arena/arena.h @@ -0,0 +1,23 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_INTERNAL_ARENA_H +#define MI_INTERNAL_ARENA_H + +#include "../types.h" + +mi_arena_id_t _mi_arena_id_none(void); +mi_arena_t* _mi_arena_from_id(mi_arena_id_t id); +bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_t* request_arena); + +void _mi_arenas_unsafe_destroy_all(mi_subproc_t* subproc); + +mi_heap_t* _mi_arena_heap_main(const mi_arena_t* arena); +size_t _mi_slice_count_of_size(size_t size); +size_t _mi_size_of_slices(size_t bcount); + +#endif diff --git a/system/lib/mimalloc/include/mimalloc/atomic.h b/system/lib/mimalloc/include/mimalloc/atomic.h index 9c23398024ab7..2cbe2486d3d6d 100644 --- a/system/lib/mimalloc/include/mimalloc/atomic.h +++ b/system/lib/mimalloc/include/mimalloc/atomic.h @@ -9,7 +9,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_ATOMIC_H // include windows.h or pthreads.h -#if defined(_WIN32) || defined(__CYGWIN__) // we use windows locks on cygwin, but otherwise treat it at unix +#if defined(_WIN32) || defined(__CYGWIN__) // we use windows locks on cygwin, but otherwise treat it as unix #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif @@ -42,7 +42,7 @@ terms of the MIT license. A copy of the license can be found in the file #else #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) #endif -#elif defined(_MSC_VER) +#elif defined(_MSC_VER) && !defined(__clang__) // Use MSVC C wrapper for C11 atomics #define _Atomic(tp) tp #define MI_ATOMIC_VAR_INIT(x) x @@ -93,16 +93,20 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_atomic_or_relaxed(p,x) mi_atomic(fetch_or_explicit)(p,x,mi_memory_order(relaxed)) #define mi_atomic_or_acq_rel(p,x) mi_atomic(fetch_or_explicit)(p,x,mi_memory_order(acq_rel)) -#define mi_atomic_increment_relaxed(p) mi_atomic_add_relaxed(p,(uintptr_t)1) -#define mi_atomic_decrement_relaxed(p) mi_atomic_sub_relaxed(p,(uintptr_t)1) -#define mi_atomic_increment_acq_rel(p) mi_atomic_add_acq_rel(p,(uintptr_t)1) -#define mi_atomic_decrement_acq_rel(p) mi_atomic_sub_acq_rel(p,(uintptr_t)1) +#define mi_atomic_increment_relaxed(p) mi_atomic_add_relaxed(p,(size_t)1) +#define mi_atomic_decrement_relaxed(p) mi_atomic_sub_relaxed(p,(size_t)1) +#define mi_atomic_increment_acq_rel(p) mi_atomic_add_acq_rel(p,(size_t)1) +#define mi_atomic_decrement_acq_rel(p) mi_atomic_sub_acq_rel(p,(size_t)1) -static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add); -static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub); +static inline mi_ssize_t mi_atomic_addi_acq_rel(_Atomic(mi_ssize_t)*p, mi_ssize_t add); +static inline mi_ssize_t mi_atomic_subi_acq_rel(_Atomic(mi_ssize_t)*p, mi_ssize_t sub); +static inline mi_ssize_t mi_atomic_addi_relaxed(_Atomic(mi_ssize_t)*p, mi_ssize_t add); +static inline mi_ssize_t mi_atomic_subi_relaxed(_Atomic(mi_ssize_t)*p, mi_ssize_t sub); +static inline void mi_atomic_storess_relaxed(_Atomic(mi_ssize_t)* p, mi_ssize_t x); +static inline mi_ssize_t mi_atomic_loads_relaxed(_Atomic(mi_ssize_t)* p); -#if defined(__cplusplus) || !defined(_MSC_VER) +#if defined(__cplusplus) || !defined(_MSC_VER) || defined(__clang__) // In C++/C11 atomics we have polymorphic atomics so can use the typed `ptr` variants (where `tp` is the type of atomic value) // We use these macros so we can provide a typed wrapper in MSVC in C compilation mode as well @@ -133,16 +137,16 @@ static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub); #endif // These are used by the statistics -static inline int64_t mi_atomic_addi64_relaxed(volatile int64_t* p, int64_t add) { +static inline int64_t mi_atomic_volatile_addi64_relaxed(volatile int64_t* p, int64_t add) { return mi_atomic(fetch_add_explicit)((_Atomic(int64_t)*)p, add, mi_memory_order(relaxed)); } -static inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const volatile int64_t* padd) { +static inline void mi_atomic_volatile_void_addi64_relaxed(volatile int64_t* p, const volatile int64_t* padd) { const int64_t add = mi_atomic_load_relaxed((_Atomic(int64_t)*)padd); if (add != 0) { mi_atomic(fetch_add_explicit)((_Atomic(int64_t)*)p, add, mi_memory_order(relaxed)); } } -static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) { +static inline void mi_atomic_volatile_maxi64_relaxed(volatile int64_t* p, int64_t x) { int64_t current = mi_atomic_load_relaxed((_Atomic(int64_t)*)p); while (current < x && !mi_atomic_cas_weak_release((_Atomic(int64_t)*)p, ¤t, x)) { /* nothing */ }; } @@ -155,9 +159,10 @@ static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) { #define mi_atomic_casi64_strong_acq_rel(p,e,d) mi_atomic_cas_strong_acq_rel(p,e,d) #define mi_atomic_addi64_acq_rel(p,i) mi_atomic_add_acq_rel(p,i) +#define mi_atomic_addi64_relaxed(p,i) mi_atomic_add_relaxed(p,i) -#elif defined(_MSC_VER) +#elif defined(_MSC_VER) && !defined(__clang__) // Deprecated: MSVC plain C compilation wrapper that uses Interlocked operations to model C11 atomics. // It is recommended to always compile as C++ when using MSVC. @@ -312,9 +317,9 @@ static inline void mi_atomic_storei64_explicit(_Atomic(int64_t)*p, int64_t x, mi } // These are used by the statistics -static inline int64_t mi_atomic_addi64_relaxed(volatile _Atomic(int64_t)*p, int64_t add) { +static inline int64_t mi_atomic_volatile_addi64_relaxed(volatile _Atomic(int64_t)*p, int64_t add) { #ifdef _WIN64 - return (int64_t)mi_atomic_addi((int64_t*)p, add); + return (int64_t)mi_atomic_addi_relaxed((_Atomic(int64_t)*)p, add); #elif defined(_M_ARM) return _InterlockedExchangeAdd64(p, add); #else @@ -329,14 +334,14 @@ static inline int64_t mi_atomic_addi64_relaxed(volatile _Atomic(int64_t)*p, int6 #endif } -static inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const volatile int64_t* padd) { +static inline void mi_atomic_volatile_void_addi64_relaxed(volatile int64_t* p, const volatile int64_t* padd) { const int64_t add = *padd; if (add != 0) { - mi_atomic_addi64_relaxed((volatile _Atomic(int64_t)*)p, add); + mi_atomic_volatile_addi64_relaxed((volatile _Atomic(int64_t)*)p, add); } } -static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)* p, int64_t x) { +static inline void mi_atomic_volatile_maxi64_relaxed(volatile _Atomic(int64_t)* p, int64_t x) { int64_t current; do { current = *p; @@ -344,7 +349,11 @@ static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)* p, int64_ } static inline void mi_atomic_addi64_acq_rel(volatile _Atomic(int64_t)* p, int64_t i) { - mi_atomic_addi64_relaxed(p, i); + mi_atomic_volatile_addi64_relaxed(p, i); +} + +static inline int64_t mi_atomic_addi64_relaxed(volatile _Atomic(int64_t)* p, int64_t i) { + return mi_atomic_volatile_addi64_relaxed(p, i); } static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t)* p, int64_t* exp, int64_t des) { @@ -379,17 +388,24 @@ static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t)* p, #endif - -// Atomically add a signed value; returns the previous value. -static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add) { - return (intptr_t)mi_atomic_add_acq_rel((_Atomic(uintptr_t)*)p, (uintptr_t)add); +static inline mi_ssize_t mi_atomic_addi_acq_rel(_Atomic(mi_ssize_t)*p, mi_ssize_t add) { + return (mi_ssize_t)mi_atomic_add_acq_rel((_Atomic(size_t)*)p, (size_t)add); } - -// Atomically subtract a signed value; returns the previous value. -static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub) { - return (intptr_t)mi_atomic_addi(p, -sub); +static inline mi_ssize_t mi_atomic_subi_acq_rel(_Atomic(mi_ssize_t)*p, mi_ssize_t sub) { + return (mi_ssize_t)mi_atomic_addi_acq_rel(p, -sub); +} +static inline mi_ssize_t mi_atomic_addi_relaxed(_Atomic(mi_ssize_t)*p, mi_ssize_t add) { + return (mi_ssize_t)mi_atomic_add_relaxed((_Atomic(size_t)*)p, (size_t)add); +} +static inline mi_ssize_t mi_atomic_subi_relaxed(_Atomic(mi_ssize_t)*p, mi_ssize_t sub) { + return (mi_ssize_t)mi_atomic_addi_relaxed(p, -sub); +} +static inline void mi_atomic_storess_relaxed(_Atomic(mi_ssize_t)* p, mi_ssize_t x) { + mi_atomic_store_relaxed((_Atomic(size_t)*)p, (size_t)x); +} +static inline mi_ssize_t mi_atomic_loads_relaxed(_Atomic(mi_ssize_t)* p) { + return (mi_ssize_t)mi_atomic_load_relaxed((_Atomic(size_t)*)p); } - // ---------------------------------------------------------------------- // Guard @@ -553,12 +569,56 @@ typedef struct mi_atomic_once_s { // Returns `true` only on the first invocation, signifying we can execute an action once. // If it returns `true`, the caller should call `_mi_atomic_once_release` after performing the action. // Other threads (than the initial thread that entered) will block until `_mi_atomic_once_release` has been called. -bool _mi_atomic_once_enter(mi_atomic_once_t* once); // defined in `libc.c` -void _mi_atomic_once_release(mi_atomic_once_t* once); // defined in `libc.c` +bool _mi_atomic_once_enter(mi_atomic_once_t* once); // defined in `util/libc.c` +void _mi_atomic_once_release(mi_atomic_once_t* once); // defined in `util/libc.c` #define mi_atomic_do_once \ static mi_atomic_once_t _mi_once = { MI_ATOMIC_VAR_INIT(0), MI_LOCK_INITIALIZER }; \ for(bool _mi_exec = _mi_atomic_once_enter(&_mi_once); _mi_exec; (_mi_atomic_once_release(&_mi_once),_mi_exec=false)) + +/* ----------------------------------------------------------- + pthread thread locals +----------------------------------------------------------- */ + +#if MI_USE_PTHREADS + +#if defined(__APPLE__) && defined(__aarch64__) +#define MI_PTHREAD_KEY_INVALID ((pthread_key_t)(0)) // nicer codegen +#else +#define MI_PTHREAD_KEY_INVALID ((pthread_key_t)(-1)) +#endif + +#if defined(__linux__) && defined(__GLIBC__) +// pthread_getspecific returns NULL for invalid keys. +// see also: +#define MI_PTHREADS_GET_INVALID_KEY_IS_NULL 1 +#endif + +mi_decl_noinline bool _mi_pthread_key_create(pthread_key_t* pkey, void (*destruct)(void*), void* init); + +static inline void* mi_pthread_key_get(pthread_key_t key) { + #if !MI_PTHREADS_GET_INVALID_KEY_IS_NULL + if mi_unlikely(key==MI_PTHREAD_KEY_INVALID) return NULL; + #endif + return pthread_getspecific(key); +} + +static inline bool mi_pthread_key_set(pthread_key_t* pkey, void* val) { + if mi_likely(*pkey!=MI_PTHREAD_KEY_INVALID) { pthread_setspecific(*pkey,val); return true; } + else if (val!=NULL) { return _mi_pthread_key_create(pkey,NULL,val); } + else return true; +} + +static inline void mi_pthread_key_delete(pthread_key_t* pkey) { + const pthread_key_t key = *pkey; + if (key!=MI_PTHREAD_KEY_INVALID) { + *pkey = MI_PTHREAD_KEY_INVALID; + pthread_key_delete(key); + } +} +#endif + + #endif // __MIMALLOC_ATOMIC_H diff --git a/system/lib/mimalloc/include/mimalloc/bits.h b/system/lib/mimalloc/include/mimalloc/bits.h index 5aa0d83755f6c..293dc0be05080 100644 --- a/system/lib/mimalloc/include/mimalloc/bits.h +++ b/system/lib/mimalloc/include/mimalloc/bits.h @@ -57,9 +57,11 @@ terms of the MIT license. A copy of the license can be found in the file #if SIZE_MAX == UINT64_MAX # define MI_SIZE_SHIFT (3) typedef int64_t mi_ssize_t; +#define MI_SSIZE_MAX INT64_MAX #elif SIZE_MAX == UINT32_MAX # define MI_SIZE_SHIFT (2) typedef int32_t mi_ssize_t; +#define MI_SSIZE_MAX INT32_MAX #else #error platform objects must be 32 or 64 bits in size #endif @@ -125,7 +127,7 @@ typedef int32_t mi_ssize_t; // #define MI_BIG_ENDIAN 1 // maximum virtual address bits in a user-space pointer -#if MI_DEFAULT_VIRTUAL_ADDRESS_BITS > 0 +#if MI_DEFAULT_VIRTUAL_ADDRESS_BITS > 0 #define MI_MAX_VABITS MI_DEFAULT_VIRTUAL_ADDRESS_BITS #elif MI_ARCH_X64 #define MI_MAX_VABITS (47) @@ -146,7 +148,7 @@ typedef int32_t mi_ssize_t; // use a flat page-map or a 2-level one #ifndef MI_PAGE_MAP_FLAT -#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && !MI_SECURE && !MI_FREE_IS_CHECKED && MI_FREE_USE_PAGEMAP +#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && !MI_SECURE && !MI_FREE_IS_CHECKED && MI_FREE_USE_PAGEMAP #define MI_PAGE_MAP_FLAT 1 #else #define MI_PAGE_MAP_FLAT 0 @@ -227,7 +229,7 @@ static inline size_t mi_ctz(size_t x) { size_t r; __asm ("tzcnt\t%1, %0" : "=r"(r) : "r"(x) : "cc"); return r; - #elif defined(_MSC_VER) && MI_ARCH_X64 && defined(__BMI1__) + #elif defined(_MSC_VER) && MI_ARCH_X64 && defined(__BMI1__) return _tzcnt_u64(x); #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32) unsigned long idx; @@ -251,7 +253,7 @@ static inline size_t mi_clz(size_t x) { size_t r; __asm ("lzcnt\t%1, %0" : "=r"(r) : "r"(x) : "cc"); return r; - #elif defined(_MSC_VER) && MI_ARCH_X64 && defined(__LZCNT__) + #elif defined(_MSC_VER) && MI_ARCH_X64 && defined(__LZCNT__) return _lzcnt_u64(x); #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32) unsigned long idx; diff --git a/system/lib/mimalloc/include/mimalloc/decl.h b/system/lib/mimalloc/include/mimalloc/decl.h new file mode 100644 index 0000000000000..657eb7b8cc688 --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/decl.h @@ -0,0 +1,180 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2026 Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +/* ---------------------------------------------------------------------------- + Compiler dependent definitions +---------------------------------------------------------------------------- */ + +#pragma once +#ifndef MI_DECL_H +#define MI_DECL_H + +// -------------------------------------------------------------------------- +// Compiler defines +// -------------------------------------------------------------------------- + +#define mi_decl_cache_align mi_decl_align(64) + +#if defined(_MSC_VER) +#pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths) +#pragma warning(disable:26812) // unscoped enum warning +#define mi_decl_forceinline __forceinline +#define mi_decl_noinline __declspec(noinline) +#define mi_decl_thread __declspec(thread) +#define mi_decl_noreturn __declspec(noreturn) +#define mi_decl_weak +#define mi_decl_hidden +#define mi_decl_cold +#define mi_assume_aligned(p,sz) (p) +#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc +#if !MI_TRACK_ASAN +#define mi_decl_forceinline __attribute__((always_inline)) inline +#else +#define mi_decl_forceinline inline +#endif +#define mi_decl_noinline __attribute__((noinline)) +#define mi_decl_thread __thread +#define mi_decl_noreturn __attribute__((noreturn)) +#define mi_decl_weak __attribute__((weak)) +#if defined(__MINGW32__) || defined(__CYGWIN__) +#define mi_decl_hidden +#else +#define mi_decl_hidden __attribute__((visibility("hidden"))) +#endif +#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) +#define mi_decl_cold __attribute__((cold)) +#define mi_assume_aligned(p,sz) __builtin_assume_aligned(p,sz) +#else +#define mi_decl_cold +#define mi_assume_aligned(p,sz) (p) +#endif +#elif __cplusplus >= 201103L // c++11 +#define mi_decl_forceinline inline +#define mi_decl_noinline +#define mi_decl_thread thread_local +#define mi_decl_noreturn [[noreturn]] +#define mi_decl_weak +#define mi_decl_hidden +#define mi_decl_cold +#define mi_assume_aligned(p,sz) (p) +#else +#define mi_decl_forceinline inline +#define mi_decl_noinline +#define mi_decl_thread __thread // hope for the best :-) +#define mi_decl_noreturn +#define mi_decl_weak +#define mi_decl_hidden +#define mi_decl_cold +#define mi_assume_aligned(p,sz) (p) +#endif + +#if defined(__GNUC__) || defined(__clang__) +#define mi_unlikely(x) (__builtin_expect(!!(x),false)) +#define mi_likely(x) (__builtin_expect(!!(x),true)) +#elif (defined(__cplusplus) && (__cplusplus >= 202002L)) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L) +#define mi_unlikely(x) (x) [[unlikely]] +#define mi_likely(x) (x) [[likely]] +#else +#define mi_unlikely(x) (x) +#define mi_likely(x) (x) +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) +#define mi_prefetch_write(p) __builtin_prefetch(p, 1, 3) +#define mi_prefetch_read(p) __builtin_prefetch(p, 0, 3) +#else +#define mi_prefetch_write(p) +#define mi_prefetch_read(p) +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 7)) || defined(__clang__) +#define mi_decl_maybe_unused __attribute__((unused)) +#elif __cplusplus >= 201703L // c++17 +#define mi_decl_maybe_unused [[maybe_unused]] +#else +#define mi_decl_maybe_unused +#endif + +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + +#if defined(__EMSCRIPTEN__) && !defined(__wasi__) +#define __wasi__ +#endif + +#define MI_UNUSED(x) (void)(x) +#if (MI_DEBUG>1) +#define MI_UNUSED_RELEASE(x) +#else +#define MI_UNUSED_RELEASE(x) MI_UNUSED(x) +#endif + +// ------------------------------------------------------ +// Assertions and tracing +// ------------------------------------------------------ + +#if (MI_DEBUG>0) +void _mi_trace_message(const char* fmt, ...); +#define mi_trace_message(...) _mi_trace_message(__VA_ARGS__) +#else +#define mi_trace_message(...) +#endif + + +#if (MI_DEBUG) +// use our own assertion to print without memory allocation +mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func) mi_attr_noexcept; +#define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__)) +#else +#define mi_assert(x) +#endif + +#if (MI_DEBUG>1) +#define mi_assert_internal mi_assert +#else +#define mi_assert_internal(x) +#endif + +#if (MI_DEBUG>2) +#define mi_assert_expensive mi_assert +#else +#define mi_assert_expensive(x) +#endif + + +// ------------------------------------------------------ +// Initialization macros +// ------------------------------------------------------ + +#define MI_INIT4(x) x(),x(),x(),x() +#define MI_INIT8(x) MI_INIT4(x),MI_INIT4(x) +#define MI_INIT16(x) MI_INIT8(x),MI_INIT8(x) +#define MI_INIT32(x) MI_INIT16(x),MI_INIT16(x) +#define MI_INIT64(x) MI_INIT32(x),MI_INIT32(x) +#define MI_INIT128(x) MI_INIT64(x),MI_INIT64(x) +#define MI_INIT256(x) MI_INIT128(x),MI_INIT128(x) + +#define MI_INIT74(x) MI_INIT64(x),MI_INIT8(x),x(),x() +#define MI_INIT5(x) MI_INIT4(x),x() +#define MI_INIT6(x) MI_INIT4(x),x(),x() + +// ------------------------------------------------------ +// Interposing for macOS (DYLD) +// ------------------------------------------------------ + +#if defined(__APPLE__) && defined(__MACH__) +struct mi_interpose_s { + const void* replacement; + const void* target; +}; +#define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun } +#define MI_INTERPOSE_MI(fun) MI_INTERPOSE_FUN(fun,mi_##fun) +#define MI_INTERPOSE_DECLS(name) __attribute__((used)) static const struct mi_interpose_s name[] __attribute__((section("__DATA, __interpose"))) +#endif + +#endif // MI_DECL_H diff --git a/system/lib/mimalloc/include/mimalloc/heap/heap.h b/system/lib/mimalloc/include/mimalloc/heap/heap.h new file mode 100644 index 0000000000000..0a8e944942652 --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/heap/heap.h @@ -0,0 +1,48 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_INTERNAL_HEAP_H +#define MI_INTERNAL_HEAP_H + +#include "../types.h" +#include "init.h" +#include "subproc.h" + +void _mi_heap_init(mi_heap_t* heap, mi_thread_local_t theap, mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id, bool use_qsbr); +mi_decl_cold mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap); // get (and possible create) the theap belonging to a heap +void _mi_heap_force_destroy(mi_heap_t* heap, bool acquire_heaps_lock); // allow destroying the main heap +mi_heap_t* _mi_heap_new_for_subproc(mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id, bool is_heap_main, bool use_qsbr); +bool _mi_heap_theap_set(mi_heap_t* heap, mi_theap_t* theap); + +static inline bool mi_heap_qsbr_is_enabled(const mi_heap_t* heap) { +#if MI_QSBR + return heap->qsbr_enabled; +#else + MI_UNUSED(heap); + return false; +#endif +} + +static inline long mi_heap_reclaim_on_free(mi_heap_t* heap) { + return heap->reclaim_on_free; +} + +static inline mi_heap_t* mi_heap_get_heap_main(const mi_heap_t* heap) { + return _mi_subproc_heap_main(heap->subproc); +} + +static inline bool mi_is_heap_main(const mi_heap_t* heap) { + mi_assert_internal(heap!=NULL); + return (mi_heap_get_heap_main(heap) == heap); +} + +static inline bool mi_is_process_heap_main(const mi_heap_t* heap) { + mi_assert_internal(heap!=NULL); + return (_mi_subproc_main()->heap_main == heap); +} + +#endif diff --git a/system/lib/mimalloc/include/mimalloc/heap/init.h b/system/lib/mimalloc/include/mimalloc/heap/init.h new file mode 100644 index 0000000000000..3f2c6506c66b2 --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/heap/init.h @@ -0,0 +1,27 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_INTERNAL_INIT_H +#define MI_INTERNAL_INIT_H + +#include "../types.h" + +mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc); +mi_theap_t* _mi_theap_main(void); // current thread's backing theap, without initialization +mi_page_t* _mi_page_empty_get(void); +void _mi_auto_process_init(void); +void mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept; +bool _mi_preloading(void); // true while the C runtime is not initialized yet +void _mi_thread_done(mi_theap_t* theap); +mi_theap_t* _mi_thread_init(void); +mi_theap_t* _mi_thread_init_with_heap(mi_heap_t* heap); +bool _mi_is_empty_theap(const mi_theap_t* theap); + +extern mi_decl_hidden const mi_theap_t _mi_theap_empty; // read-only empty theap, initial value of the thread local default theap (in the MI_TLS_MODEL_LOCAL) +extern mi_decl_hidden mi_theap_t _mi_theap_empty_wrong; // read-only empty theap used to signal that a theap for a heap could not be allocated + +#endif diff --git a/system/lib/mimalloc/include/mimalloc/heap/page-map.h b/system/lib/mimalloc/include/mimalloc/heap/page-map.h new file mode 100644 index 0000000000000..fd26187ac8adb --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/heap/page-map.h @@ -0,0 +1,173 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_INTERNAL_PAGE_MAP_H +#define MI_INTERNAL_PAGE_MAP_H + +#include "../types.h" +#include "../util/libc.h" +#include "../util/options.h" + +bool _mi_page_map_init(void); +mi_decl_nodiscard bool _mi_page_map_register(mi_page_t* page); +void _mi_page_map_unregister(mi_page_t* page); +void _mi_page_map_unregister_range(void* start, size_t size); +void _mi_page_map_unsafe_destroy(void); +mi_page_t* _mi_safe_ptr_page(const void* p); + + +/* ----------------------------------------------------------- + The page map maps addresses to `mi_page_t` pointers +----------------------------------------------------------- */ + +#if MI_PAGE_MAP_FLAT + +// flat page-map committed on demand, using one byte per slice (64 KiB). +// single indirection and low commit, but large initial virtual reserve (4 GiB with 48 bit virtual addresses) +// used by default on <= 40 bit virtual address spaces. +extern mi_decl_hidden _Atomic(uint8_t*) _mi_page_map; +extern mi_decl_hidden _Atomic(void*) _mi_page_map_max_address; + +static inline size_t mi_page_map_index(const void* p) { + return (size_t)((uintptr_t)p >> MI_ARENA_SLICE_SHIFT); +} + +static inline uint8_t mi_page_map_at(size_t idx) { + return mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map)[idx]; +} + +static inline mi_page_t* mi_ptr_page_ex(const void* p, bool* valid) { + const size_t idx = mi_page_map_index(p); + const size_t ofs = mi_page_map_at(idx); + if (valid != NULL) { *valid = (ofs != 0); } + return (mi_page_t*)((((uintptr_t)p >> MI_ARENA_SLICE_SHIFT) + 1 - ofs) << MI_ARENA_SLICE_SHIFT); +} + +static inline mi_page_t* mi_checked_ptr_page(const void* p) { + #if MI_MIN_VABITS < MI_INTPTR_BITS + if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) { + if (p > mi_atomic_load_ptr_relaxed(void, &_mi_page_map_max_address)) return NULL; + } + #endif + bool valid; + mi_page_t* const page = mi_ptr_page_ex(p, &valid); + return (valid ? page : NULL); +} + +static inline mi_page_t* mi_unchecked_ptr_page(const void* p) { + return mi_ptr_page_ex(p, NULL); +} + +#else + +// 2-level page map: +// double indirection, but low commit and low virtual reserve. +// +// the page-map is usually 4 MiB (for 48 bit virtual addresses) and points to sub maps of 64 KiB. +// the page-map is committed on-demand (in 64 KiB parts) (and sub-maps are committed on-demand as well) +// one sub page-map = 64 KiB => covers 2^(16-3) * 2^16 = 2^29 = 512 MiB address space +// the page-map needs 48-(16+13) = 19 bits => 2^19 sub map pointers = 2^22 bytes = 4 MiB reserved size. +#define MI_PAGE_MAP_SUB_SHIFT (13) +#define MI_PAGE_MAP_SUB_COUNT (MI_ZU(1) << MI_PAGE_MAP_SUB_SHIFT) +#define MI_PAGE_MAP_SHIFT (MI_MAX_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT) + +typedef mi_page_t** mi_submap_t; +typedef struct mi_page_map_s { + _Atomic(size_t) committed_count; // currently committed entries + size_t reserved_size; // full reserved size (mi_page_map_t + submaps) + mi_memid_t memid; // provenance + mi_lock_t lock; // used when allocating new submaps + _Atomic(mi_submap_t) submaps[1]; +} mi_page_map_t; + +extern mi_decl_hidden _Atomic(mi_page_map_t*) __mi_page_map; + +static inline size_t mi_page_map_index(const void* p, size_t* sub_idx) { + const size_t u = (size_t)((uintptr_t)p / MI_ARENA_SLICE_SIZE); + if (sub_idx != NULL) { *sub_idx = u % MI_PAGE_MAP_SUB_COUNT; } + return (u / MI_PAGE_MAP_SUB_COUNT); +} + +static inline mi_page_map_t* mi_page_map(void) { + return mi_atomic_load_ptr_relaxed(mi_page_map_t,&__mi_page_map); +} + +static inline mi_submap_t mi_page_map_at(const mi_page_map_t* pmap, size_t idx) { + return mi_atomic_load_ptr_acquire(mi_page_t*, &pmap->submaps[idx]); +} + +static inline mi_page_t* mi_unchecked_ptr_page(const void* p) { + const mi_page_map_t* pmap = mi_page_map(); + size_t sub_idx; + const size_t idx = mi_page_map_index(p, &sub_idx); + return mi_page_map_at(pmap,idx)[sub_idx]; // NULL if p==NULL +} + +static inline mi_page_t* mi_checked_ptr_page(const void* p) { + const mi_page_map_t* pmap = mi_page_map(); + size_t sub_idx; + const size_t idx = mi_page_map_index(p, &sub_idx); + const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); + if mi_unlikely(idx >= committed_count) return NULL; + // #if MI_MIN_VABITS < MI_INTPTR_BITS // is still invalid if free is called before the pagemap is initialized + // if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) { + // const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); + // if mi_unlikely(idx >= committed_count) return NULL; + // } + // #endif + mi_submap_t const sub = mi_page_map_at(pmap,idx); + if mi_unlikely(sub == NULL) return NULL; + return sub[sub_idx]; +} + +#endif + +#if MI_PAGE_META_IS_ALIGNED +// if the page meta data is aligned in front of pages we can find it efficiently +// without needing to go through the page map (for valid pointers). +static inline mi_page_t* mi_aligned_ptr_page0(const void* p) { + mi_page_t* const page_metas = (mi_page_t*)mi_align_down_ptr(p,MI_PAGE_META_ALIGNMENT); + // const ptrdiff_t page_idx = ((uint8_t*)p - (uint8_t*)page_metas)/MI_ARENA_SLICE_SIZE; + const uintptr_t page_idx = ((uintptr_t)p / MI_ARENA_SLICE_SIZE) % (MI_PAGE_META_ALIGNMENT / MI_ARENA_SLICE_SIZE); + mi_assert_internal(page_idx <= MI_PAGE_META_ALIGNED_COUNT); + #if MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_RISCV // better code on x64/x86/riscv64 + mi_page_t* const page = (mi_page_t*)((uintptr_t)page_metas | (page_idx * sizeof(mi_page_t))); + #else + mi_page_t* const page = &page_metas[page_idx]; + #endif + return page; + +} + +static inline mi_page_t* mi_aligned_ptr_page(const void* p) { + mi_page_t* const page = mi_aligned_ptr_page0(p); + if mi_unlikely(page==NULL) return NULL; + #if MI_DEBUG + mi_page_t* const cpage = mi_checked_ptr_page(p); + if mi_unlikely(cpage==NULL) { + _mi_error_message(EINVAL, "mi_aligned_ptr_page: invalid pointer: %p\n", p); + return NULL; + } + #endif + return mi_atomic_load_ptr_acquire(mi_page_t, &page->self); +} +#endif + +static inline mi_page_t* mi_ptr_page(const void* p) { + mi_assert_internal(p==NULL || mi_is_in_heap_region(p)); + #if MI_SECURE || MI_FREE_IS_CHECKED + return mi_checked_ptr_page(p); + #elif MI_PAGE_META_IS_ALIGNED + return mi_aligned_ptr_page(p); + #elif MI_DEBUG + return mi_checked_ptr_page(p); + #else + return mi_unchecked_ptr_page(p); + #endif +} + +#endif diff --git a/system/lib/mimalloc/include/mimalloc/heap/subproc.h b/system/lib/mimalloc/include/mimalloc/heap/subproc.h new file mode 100644 index 0000000000000..0b37a5dfb4d8d --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/heap/subproc.h @@ -0,0 +1,29 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_INTERNAL_SUBPROC_H +#define MI_INTERNAL_SUBPROC_H + +#include "../types.h" + +mi_subproc_t* _mi_subproc_main_init(void); +void _mi_subproc_main_done(void); +mi_subproc_t* _mi_subproc_main(void); +bool _mi_subproc_is_main(mi_subproc_t* subproc); +mi_subproc_t* _mi_subproc(void); // current subproc of this thread +mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id); +void _mi_subprocs_unsafe_destroy_all(void); +void _mi_fork_init(void); +void _mi_fork_child(void); + +void* _mi_meta_zalloc( mi_subproc_t* subproc, size_t size, mi_memid_t* memid ); +void* _mi_meta_rezalloc( mi_subproc_t* subproc, void* p, size_t newsize, mi_memid_t* memid ); +void* _mi_meta_zalloc_aligned( mi_subproc_t* subproc, size_t size, size_t alignment, mi_memid_t* memid ); +void _mi_meta_free(mi_subproc_t* subproc, void* p, mi_memid_t memid); +bool _mi_meta_is_meta_page(const mi_subproc_t* subproc, const mi_page_t* p); + +#endif diff --git a/system/lib/mimalloc/include/mimalloc/heap/theap.h b/system/lib/mimalloc/include/mimalloc/heap/theap.h new file mode 100644 index 0000000000000..86f42b8a82e25 --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/heap/theap.h @@ -0,0 +1,65 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_INTERNAL_THEAP_H +#define MI_INTERNAL_THEAP_H + +#include "../types.h" + +size_t _mi_theap_random_next(mi_theap_t* theap); +void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld); +mi_theap_t* _mi_theap_alloc(mi_heap_t* heap, mi_tld_t* tld); +mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld); +void _mi_theap_collect_abandon(mi_theap_t* theap); +void _mi_theap_abandon_owned_pages(mi_theap_t* theap); +bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg); + +void _mi_heap_detach_theaps( mi_heap_t* heap ); +void _mi_tld_detach_theaps( mi_tld_t* tld ); +void _mi_theap_incref(mi_theap_t* theap); +void _mi_theap_decref(mi_theap_t* theap); +void _mi_theap_merge_stats(mi_theap_t* theap); + +void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page); + +/*---------------------------------------------------------------------------------------- + Heap functions +------------------------------------------------------------------------------------------- */ + +static inline mi_heap_t* mi_theap_heap_peek(const mi_theap_t* theap) { + mi_assert_internal(theap!=NULL); + return mi_atomic_load_ptr_relaxed(mi_heap_t,&theap->heap); +} + +static inline mi_heap_t* mi_theap_heap(const mi_theap_t* theap) { + mi_heap_t* const heap = mi_theap_heap_peek(theap); + mi_assert_internal(heap!=NULL); + return heap; +} + +static inline bool mi_theap_is_initialized(const mi_theap_t* theap) { + return (theap != NULL && mi_theap_heap_peek(theap) != NULL); +} + +static inline mi_subproc_t* mi_theap_subproc(const mi_theap_t* theap) { + mi_subproc_t* const subproc = mi_atomic_load_ptr_relaxed(mi_subproc_t,&theap->subproc); + mi_assert_internal(!mi_theap_is_initialized(theap) || mi_theap_heap(theap)->subproc == subproc); + return subproc; +} + +static inline mi_page_t* mi_theap_get_free_small_page(mi_theap_t* theap, size_t xsize, bool is_wsize) { + mi_assert_internal(is_wsize ? xsize <= (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE) : xsize <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE)); + const size_t idx = (is_wsize ? xsize : mi_wsize_from_size(xsize)); + mi_assert_internal(idx < MI_PAGES_DIRECT); + return theap->pages_free_direct[idx]; +} + +static inline bool mi_theap_is_detached(mi_theap_t* theap) { + return (theap!=NULL && theap->tld->thread_id == MI_THREADID_DETACHED); +} + +#endif diff --git a/system/lib/mimalloc/include/mimalloc/internal.h b/system/lib/mimalloc/include/mimalloc/internal.h index b79649c728005..2f1ecf489edce 100644 --- a/system/lib/mimalloc/include/mimalloc/internal.h +++ b/system/lib/mimalloc/include/mimalloc/internal.h @@ -9,1547 +9,42 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_INTERNAL_H // -------------------------------------------------------------------------- -// This file contains the internal API's of mimalloc and various utility -// functions and macros. +// This file includes all the internal API's of mimalloc for convenience. +// (but is not used by mimalloc itself) // -------------------------------------------------------------------------- #include "types.h" #include "track.h" -#include "bits.h" - - -// -------------------------------------------------------------------------- -// Compiler defines -// -------------------------------------------------------------------------- - -#if (MI_DEBUG>0) -#define mi_trace_message(...) _mi_trace_message(__VA_ARGS__) -#else -#define mi_trace_message(...) -#endif - -#define mi_decl_cache_align mi_decl_align(64) - -#if defined(_MSC_VER) -#pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths) -#pragma warning(disable:26812) // unscoped enum warning -#define mi_decl_forceinline __forceinline -#define mi_decl_noinline __declspec(noinline) -#define mi_decl_thread __declspec(thread) -#define mi_decl_noreturn __declspec(noreturn) -#define mi_decl_weak -#define mi_decl_hidden -#define mi_decl_cold -#define mi_assume_aligned(p,sz) (p) -#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc -#if !MI_TRACK_ASAN -#define mi_decl_forceinline __attribute__((always_inline)) inline -#else -#define mi_decl_forceinline inline -#endif -#define mi_decl_noinline __attribute__((noinline)) -#define mi_decl_thread __thread -#define mi_decl_noreturn __attribute__((noreturn)) -#define mi_decl_weak __attribute__((weak)) -#if defined(__MINGW32__) || defined(__CYGWIN__) -#define mi_decl_hidden -#else -#define mi_decl_hidden __attribute__((visibility("hidden"))) -#endif -#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) -#define mi_decl_cold __attribute__((cold)) -#define mi_assume_aligned(p,sz) __builtin_assume_aligned(p,sz) -#else -#define mi_decl_cold -#define mi_assume_aligned(p,sz) (p) -#endif -#elif __cplusplus >= 201103L // c++11 -#define mi_decl_forceinline inline -#define mi_decl_noinline -#define mi_decl_thread thread_local -#define mi_decl_noreturn [[noreturn]] -#define mi_decl_weak -#define mi_decl_hidden -#define mi_decl_cold -#define mi_assume_aligned(p,sz) (p) -#else -#define mi_decl_forceinline inline -#define mi_decl_noinline -#define mi_decl_thread __thread // hope for the best :-) -#define mi_decl_noreturn -#define mi_decl_weak -#define mi_decl_hidden -#define mi_decl_cold -#define mi_assume_aligned(p,sz) (p) -#endif - -#if defined(__GNUC__) || defined(__clang__) -#define mi_unlikely(x) (__builtin_expect(!!(x),false)) -#define mi_likely(x) (__builtin_expect(!!(x),true)) -#elif (defined(__cplusplus) && (__cplusplus >= 202002L)) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L) -#define mi_unlikely(x) (x) [[unlikely]] -#define mi_likely(x) (x) [[likely]] -#else -#define mi_unlikely(x) (x) -#define mi_likely(x) (x) -#endif - -#if (defined(__GNUC__) && (__GNUC__ >= 7)) || defined(__clang__) -#define mi_decl_maybe_unused __attribute__((unused)) -#elif __cplusplus >= 201703L // c++17 -#define mi_decl_maybe_unused [[maybe_unused]] -#else -#define mi_decl_maybe_unused -#endif - -#ifndef __has_builtin -#define __has_builtin(x) 0 -#endif - -#if defined(__EMSCRIPTEN__) && !defined(__wasi__) -#define __wasi__ -#endif - // -------------------------------------------------------------------------- // Internal functions // -------------------------------------------------------------------------- +#include "prim/os.h" +#include "prim/prim-tls.h" -// "libc.c" -#include -int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args); -int _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...); -char _mi_toupper(char c); -int _mi_strnicmp(const char* s, const char* t, size_t n); -bool _mi_strlcpy(char* dest, const char* src, size_t dest_size); // returns true if the entire src was copied -bool _mi_strlcat(char* dest, const char* src, size_t dest_size); // returns true if the entire src was appended -size_t _mi_strlen(const char* s); -size_t _mi_strnlen(const char* s, size_t max_len); -char* _mi_strnstr(char* s, size_t max_len, const char* pat); -bool _mi_streq(const char* s, const char* t); -int _mi_getenv(const char* name, char* result, size_t result_size); -void _mi_detect_cpu_features(void); - -// "options.c" -void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message); -void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...); -void _mi_raw_message(const char* fmt, ...); -void _mi_warning_message(const char* fmt, ...); -void _mi_verbose_message(const char* fmt, ...); -void _mi_trace_message(const char* fmt, ...); -void _mi_options_init(void); -void _mi_options_post_init(void); -long _mi_option_get_fast(mi_option_t option); -void _mi_error_message(int err, const char* fmt, ...); - -// random.c -void _mi_random_init(mi_random_ctx_t* ctx); -void _mi_random_init_weak(mi_random_ctx_t* ctx); -void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx); -void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx); -size_t _mi_random_next(mi_random_ctx_t* ctx); -size_t _mi_theap_random_next(mi_theap_t* theap); -size_t _mi_os_random_weak(size_t extra_seed); -static inline size_t _mi_random_shuffle(size_t x); - -// prim-tls.c -void _mi_tls_slots_init(void); -void _mi_tls_slots_done(void); -mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; -void _mi_theap_default_set(mi_theap_t* theap); -void _mi_theap_cached_set(mi_theap_t* theap); - -// subproc.c -mi_subproc_t* _mi_subproc_main_init(void); -void _mi_subproc_main_done(void); -mi_subproc_t* _mi_subproc_main(void); -bool _mi_subproc_is_main(mi_subproc_t* subproc); -mi_subproc_t* _mi_subproc(void); // current subproc of this thread -mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc); -mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id); -void _mi_subprocs_unsafe_destroy_all(void); - -void* _mi_meta_zalloc( mi_subproc_t* subproc, size_t size, mi_memid_t* memid ); -void* _mi_meta_rezalloc( mi_subproc_t* subproc, void* p, size_t newsize, mi_memid_t* memid ); -void* _mi_meta_zalloc_aligned( mi_subproc_t* subproc, size_t size, size_t alignment, mi_memid_t* memid ); -void _mi_meta_free(mi_subproc_t* subproc, void* p, mi_memid_t memid); -bool _mi_meta_is_meta_page(const mi_subproc_t* subproc, const mi_page_t* p); - - -// init.c -mi_page_t* _mi_page_empty_get(void); -void _mi_auto_process_init(void); -void mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept; -bool _mi_is_redirected(void); -bool _mi_allocator_init(const char** message); -void _mi_allocator_done(void); -bool _mi_preloading(void); // true while the C runtime is not initialized yet -void _mi_thread_done(mi_theap_t* theap); -mi_theap_t* _mi_thread_init(void); -mi_theap_t* _mi_thread_init_with_heap(mi_heap_t* heap); -bool _mi_is_empty_theap(const mi_theap_t* theap); - -// os.c -void _mi_os_init(void); // called from process init -void* _mi_os_alloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid); -void* _mi_os_zalloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid); -void _mi_os_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid); -void _mi_os_free_ex(mi_subproc_t* subproc, void* p, size_t size, bool still_committed, mi_memid_t memid ); - -size_t _mi_os_page_size(void); -size_t _mi_os_guard_page_size(void); -size_t _mi_os_good_alloc_size(size_t size); -bool _mi_os_has_overcommit(void); -bool _mi_os_has_virtual_reserve(void); -bool _mi_os_canuse_thp(void); -size_t _mi_os_virtual_address_bits(void); -size_t _mi_os_minimal_purge_size(void); - -bool _mi_os_reset(mi_subproc_t* subproc, void* addr, size_t size); -bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size); -void _mi_os_reuse(mi_subproc_t* subproc, void* p, size_t size); -mi_decl_nodiscard bool _mi_os_commit(mi_subproc_t* subproc, void* p, size_t size, bool* is_zero); -mi_decl_nodiscard bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero, size_t stat_already_committed); -mi_decl_nodiscard bool _mi_os_protect(void* addr, size_t size); -bool _mi_os_unprotect(void* addr, size_t size); -bool _mi_os_purge(mi_subproc_t* subproc, void* p, size_t size); -bool _mi_os_purge_ex(mi_subproc_t* subproc, void* p, size_t size, bool allow_reset, size_t stats_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg); - -size_t _mi_os_secure_guard_page_size(void); -bool _mi_os_secure_guard_page_set_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid); -bool _mi_os_secure_guard_page_set_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid); -bool _mi_os_secure_guard_page_reset_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid); -bool _mi_os_secure_guard_page_reset_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid); - -int _mi_os_numa_node(void); -int _mi_os_numa_node_count(void); - -void* _mi_os_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid); -void* _mi_os_alloc_aligned_at_offset(mi_subproc_t* subproc, size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid); - -void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size); -bool _mi_os_canuse_large_page(size_t size, size_t alignment); -size_t _mi_os_large_page_size(void); -void* _mi_os_alloc_huge_os_pages(mi_subproc_t* subproc, size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid); - -// threadlocal.c -#define mi_thread_local_key_fast ((mi_thread_local_t)1) - -mi_thread_local_t _mi_thread_local_create(void); -void _mi_thread_local_free( mi_thread_local_t key ); -bool _mi_thread_local_set( mi_thread_local_t key, void* val ); -void* _mi_thread_local_get( mi_thread_local_t key ); -void _mi_thread_locals_init(void); -void _mi_thread_locals_done(void); -void _mi_thread_locals_thread_done(void); - -// arena.c -mi_arena_id_t _mi_arena_id_none(void); -mi_arena_t* _mi_arena_from_id(mi_arena_id_t id); -bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_t* request_arena); - -void* _mi_arenas_alloc(mi_heap_t* heap, size_t size, bool commit, bool allow_pinned, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid); -void* _mi_arenas_alloc_aligned(mi_heap_t* heap, size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_pinned, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid); -void _mi_arenas_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid); -void _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld); -void _mi_arenas_unsafe_destroy_all(mi_subproc_t* subproc); - -mi_page_t* _mi_arenas_page_alloc(mi_theap_t* theap, size_t block_size, size_t page_alignment); -void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx /* can be NULL */); -void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap); -void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx /* can be NULL */); -bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page); - -// "page-map.c" -bool _mi_page_map_init(void); -mi_decl_nodiscard bool _mi_page_map_register(mi_page_t* page); -void _mi_page_map_unregister(mi_page_t* page); -void _mi_page_map_unregister_range(void* start, size_t size); -mi_page_t* _mi_safe_ptr_page(const void* p); -void _mi_page_map_unsafe_destroy(void); - -// "page.c" -void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; -void* _mi_malloc_generic_no_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; - -void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks -void _mi_page_unfull(mi_page_t* page); -void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq); // free the page -void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread... -void _mi_deferred_free(mi_theap_t* theap, bool force); -void _mi_page_free_collect(mi_page_t* page, bool force); -mi_block_t* _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head); -mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page); -bool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq); -void _mi_page_update_stats(mi_page_t* page); - -size_t _mi_page_stats_bin(const mi_page_t* page); // for stats -size_t _mi_bin_size(size_t bin); // for stats -size_t _mi_bin(size_t size); // for stats - -// "theap.c" -void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld); -mi_theap_t* _mi_theap_alloc(mi_heap_t* heap, mi_tld_t* tld); -mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld); -void _mi_theap_collect_retired(mi_theap_t* theap, bool force); -void _mi_theap_collect_abandon(mi_theap_t* theap); -bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg); -void _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page); - -void _mi_heap_detach_theaps( mi_heap_t* heap ); -void _mi_tld_detach_theaps( mi_tld_t* tld ); -void _mi_theap_incref(mi_theap_t* theap); -void _mi_theap_decref(mi_theap_t* theap); -void _mi_theap_merge_stats(mi_theap_t* theap); - -// "heap.c" -void _mi_heap_init(mi_heap_t* heap, mi_thread_local_t theap, mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id); -void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page); -mi_decl_cold mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap); // get (and possible create) the theap belonging to a heap -void _mi_heap_move_pages(mi_heap_t* heap_from, mi_heap_t* heap_to); // in "arena.c" -void _mi_heap_destroy_pages(mi_heap_t* heap_from); // in "arena.c" -void _mi_heap_force_destroy(mi_heap_t* heap, bool acquire_heaps_lock); // allow destroying the main heap -mi_heap_t* _mi_heap_new_for_subproc(mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id, bool is_heap_main); -bool _mi_heap_theap_set(mi_heap_t* heap, mi_theap_t* theap); - -// "stats.c" -void _mi_stats_init(void); -void _mi_stats_merge_into(mi_stats_t* to, mi_stats_t* from); - -mi_msecs_t _mi_clock_now(void); -mi_msecs_t _mi_clock_end(mi_msecs_t start); -mi_msecs_t _mi_clock_start(void); - -// "alloc.c" -void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` -void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` -void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero) mi_attr_noexcept; -mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p); -void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size); - - -// "free.c" -void _mi_free_subproc_safe(void* p) mi_attr_noexcept; -void _mi_page_unguard_all(mi_page_t* page); -size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexcept; - -#if MI_DEBUG>1 -bool _mi_page_is_valid(mi_page_t* page); -#endif - -// "sample-guarded.c" -mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; -void _mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p); -void _mi_page_unguard_all(mi_page_t* page); -void _mi_theap_guarded_init(mi_theap_t* theap); - -// "sample-profile.c" -mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage) mi_attr_noexcept; -size_t _mi_theap_update_sample_rate(mi_theap_t* theap); - -mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, uint64_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept; -void _mi_page_profile_on_free(mi_page_t* page, mi_block_t* block, void* p); -size_t _mi_theap_set_profile_sample_rate(mi_theap_t* theap, size_t sample_rate); - - -// ------------------------------------------------------ -// Assertions -// ------------------------------------------------------ - -#if (MI_DEBUG) -// use our own assertion to print without memory allocation -mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func) mi_attr_noexcept; -#define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__)) -#else -#define mi_assert(x) -#endif - -#if (MI_DEBUG>1) -#define mi_assert_internal mi_assert -#else -#define mi_assert_internal(x) -#endif - -#if (MI_DEBUG>2) -#define mi_assert_expensive mi_assert -#else -#define mi_assert_expensive(x) -#endif - - -/* ----------------------------------------------------------- - Statistics (in `stats.c`) ------------------------------------------------------------ */ - -// add to stat keeping track of the peak -void __mi_stat_increase(mi_stat_count_t* stat, uint64_t amount); -void __mi_stat_decrease(mi_stat_count_t* stat, uint64_t amount); -void __mi_stat_increase_mt(mi_stat_count_t* stat, uint64_t amount); -void __mi_stat_decrease_mt(mi_stat_count_t* stat, uint64_t amount); - -// adjust stat in special cases to compensate for double counting (and does not adjust peak values and can decrease the total) -void __mi_stat_adjust_increase(mi_stat_count_t* stat, uint64_t amount); -void __mi_stat_adjust_decrease(mi_stat_count_t* stat, uint64_t amount); -void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, uint64_t amount); -void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, uint64_t amount); - -// counters can just be increased -static inline void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, uint64_t amount) { - mi_assert_internal(amount<=INT64_MAX); - mi_atomic_addi64_relaxed(&stat->total, (int64_t)amount); -} - -static inline void __mi_stat_counter_increase(mi_stat_counter_t* stat, uint64_t amount) { - mi_assert_internal(amount<=INT64_MAX); - stat->total += (int64_t)amount; -} - -static inline void __mi_stat_counter_decrease(mi_stat_counter_t* stat, uint64_t amount) { - mi_assert_internal(amount<=INT64_MAX); - stat->total -= (int64_t)amount; -} - -#define mi_heap_stat_counter_increase(heap,stat,amount) __mi_stat_counter_increase_mt( &(heap)->stats.stat, amount) -#define mi_heap_stat_increase(heap,stat,amount) __mi_stat_increase_mt( &(heap)->stats.stat, amount) -#define mi_heap_stat_decrease(heap,stat,amount) __mi_stat_decrease_mt( &(heap)->stats.stat, amount) -#define mi_heap_stat_adjust_increase(heap,stat,amnt) __mi_stat_adjust_increase_mt( &(heap)->stats.stat, amnt) -#define mi_heap_stat_adjust_decrease(heap,stat,amnt) __mi_stat_adjust_decrease_mt( &(heap)->stats.stat, amnt) - -#define mi_subproc_stat_counter_increase(subproc,stat,amount) __mi_stat_counter_increase_mt( &(subproc)->stats.stat, amount) -#define mi_subproc_stat_increase(subproc,stat,amount) __mi_stat_increase_mt( &(subproc)->stats.stat, amount) -#define mi_subproc_stat_decrease(subproc,stat,amount) __mi_stat_decrease_mt( &(subproc)->stats.stat, amount) -#define mi_subproc_stat_adjust_increase(subproc,stat,amount) __mi_stat_adjust_increase_mt( &(subproc)->stats.stat, amount) -#define mi_subproc_stat_adjust_decrease(subproc,stat,amount) __mi_stat_adjust_decrease_mt( &(subproc)->stats.stat, amount) - -#define mi_theap_stat_counter_increase(theap,stat,amount) __mi_stat_counter_increase( &(theap)->stats.stat, amount) -#define mi_theap_stat_counter_decrease(theap,stat,amount) __mi_stat_counter_decrease( &(theap)->stats.stat, amount) -#define mi_theap_stat_increase(theap,stat,amount) __mi_stat_increase( &(theap)->stats.stat, amount) -#define mi_theap_stat_decrease(theap,stat,amount) __mi_stat_decrease( &(theap)->stats.stat, amount) -#define mi_theap_stat_adjust_increase(theap,stat,amnt) __mi_stat_adjust_increase( &(theap)->stats.stat, amnt) -#define mi_theap_stat_adjust_decrease(theap,stat,amnt) __mi_stat_adjust_decrease( &(theap)->stats.stat, amnt) - -#define mi_theapx_stat_counter_increase(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_counter_increase(theap,stat,amount); } else { mi_heap_stat_counter_increase(heap,stat,amount); } -#define mi_theapx_stat_adjust_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_adjust_decrease(theap,stat,amount); } else { mi_heap_stat_adjust_decrease(heap,stat,amount); } -#define mi_theapx_stat_increase(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_increase(theap,stat,amount); } else { mi_heap_stat_increase(heap,stat,amount); } -#define mi_theapx_stat_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_decrease(theap,stat,amount); } else { mi_heap_stat_decrease(heap,stat,amount); } - - -/* ----------------------------------------------------------- - pthread thread locals ------------------------------------------------------------ */ - -#if MI_USE_PTHREADS - -#if defined(__APPLE__) && defined(__aarch64__) -#define MI_PTHREAD_KEY_INVALID ((pthread_key_t)(0)) // nicer codegen -#else -#define MI_PTHREAD_KEY_INVALID ((pthread_key_t)(-1)) -#endif - -#if defined(__linux__) && defined(__GLIBC__) -// pthread_getspecific returns NULL for invalid keys. -// see also: -#define MI_PTHREADS_GET_INVALID_KEY_IS_NULL 1 -#endif - -mi_decl_noinline bool _mi_pthread_key_create(pthread_key_t* pkey, void (*destruct)(void*), void* init); - -static inline void* mi_pthread_key_get(pthread_key_t key) { - #if !MI_PTHREADS_GET_INVALID_KEY_IS_NULL - if mi_unlikely(key==MI_PTHREAD_KEY_INVALID) return NULL; - #endif - return pthread_getspecific(key); -} - -static inline bool mi_pthread_key_set(pthread_key_t* pkey, void* val) { - if mi_likely(*pkey!=MI_PTHREAD_KEY_INVALID) { pthread_setspecific(*pkey,val); return true; } - else if (val!=NULL) { return _mi_pthread_key_create(pkey,NULL,val); } - else return true; -} - -static inline void mi_pthread_key_delete(pthread_key_t* pkey) { - const pthread_key_t key = *pkey; - if (key!=MI_PTHREAD_KEY_INVALID) { - *pkey = MI_PTHREAD_KEY_INVALID; - pthread_key_delete(key); - } -} -#endif - -/* ----------------------------------------------------------- - Options (exposed for the debugger) ------------------------------------------------------------ */ -typedef enum mi_option_init_e { - MI_OPTION_UNINIT, // not yet initialized - MI_OPTION_DEFAULTED, // not found in the environment, use default value - MI_OPTION_INITIALIZED // found in environment or set explicitly -} mi_option_init_t; - -typedef struct mi_option_desc_s { - long value; // the value - mi_option_init_t init; // is it initialized yet? (from the environment) - mi_option_t option; // for debugging: the option index should match the option - const char* name; // option name without `mimalloc_` prefix - const char* legacy_name; // potential legacy option name -} mi_option_desc_t; - - - -/* ----------------------------------------------------------- - Inlined definitions ------------------------------------------------------------ */ -#define MI_UNUSED(x) (void)(x) -#if (MI_DEBUG>1) -#define MI_UNUSED_RELEASE(x) -#else -#define MI_UNUSED_RELEASE(x) MI_UNUSED(x) -#endif - -#define MI_INIT4(x) x(),x(),x(),x() -#define MI_INIT8(x) MI_INIT4(x),MI_INIT4(x) -#define MI_INIT16(x) MI_INIT8(x),MI_INIT8(x) -#define MI_INIT32(x) MI_INIT16(x),MI_INIT16(x) -#define MI_INIT64(x) MI_INIT32(x),MI_INIT32(x) -#define MI_INIT128(x) MI_INIT64(x),MI_INIT64(x) -#define MI_INIT256(x) MI_INIT128(x),MI_INIT128(x) - -#define MI_INIT74(x) MI_INIT64(x),MI_INIT8(x),x(),x() -#define MI_INIT5(x) MI_INIT4(x),x() -#define MI_INIT6(x) MI_INIT4(x),x(),x() - -#include -// initialize a local variable to zero; use memset as compilers optimize constant sized memset's -#define _mi_memzero_var(x) memset(&x,0,sizeof(x)) - -// minimum -static inline size_t mi_min(size_t x, size_t y) { return (x <= y ? x : y); } - -// maximum -static inline size_t mi_max(size_t x, size_t y) { return (x >= y ? x : y); } - -// Is `x` a power of two? (0 is considered a power of two) -static inline bool _mi_is_power_of_two(uintptr_t x) { - return ((x & (x - 1)) == 0); -} - -// valid alignment values are as posix memalign: -static inline bool mi_alignment_is_valid(size_t alignment) { - return ((alignment!=0) && _mi_is_power_of_two(alignment)); -} - -// Is a pointer aligned? -static inline bool _mi_is_aligned(const void* p, size_t alignment) { - return (alignment==0 || ((uintptr_t)p % alignment) == 0); -} - -// Align upwards -static inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) { - mi_assert_internal(alignment != 0); - const uintptr_t mask = alignment - 1; - if ((alignment & mask) == 0) { // power of two? - return ((sz + mask) & ~mask); - } - else { - return (((sz + mask)/alignment)*alignment); - } -} - -// Align a pointer upwards -static inline void* _mi_align_up_ptr(const void* p, size_t alignment) { - return (void*)_mi_align_up((uintptr_t)p, alignment); -} - -// Align down -static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) { - mi_assert_internal(alignment != 0); - const uintptr_t mask = alignment - 1; - if ((alignment & mask) == 0) { // power of two? - return (sz & ~mask); - } - else { - return ((sz/alignment)*alignment); - } -} - -// Align a pointer downwards -static inline void* _mi_align_down_ptr(const void* p, size_t alignment) { - return (void*)_mi_align_down((uintptr_t)p, alignment); -} - -// Divide upwards: `s <= _mi_divide_up(s,d)*d < s+d`. -static inline uintptr_t _mi_divide_up(uintptr_t size, size_t divider) { - mi_assert_internal(divider != 0); - return (divider == 0 ? size : ((size + divider - 1) / divider)); -} - - -// clamp an integer -static inline size_t _mi_clamp(size_t sz, size_t min, size_t max) { - if (sz < min) return min; - else if (sz > max) return max; - else return sz; -} - -// Is memory zero initialized? -static inline bool mi_mem_is_zero(const void* p, size_t size) { - for (size_t i = 0; i < size; i++) { - if (((uint8_t*)p)[i] != 0) return false; - } - return true; -} - -// Overflow detecting multiply -#if __has_builtin(__builtin_umul_overflow) || (defined(__GNUC__) && (__GNUC__ >= 5)) -#include // UINT_MAX, ULONG_MAX -#if defined(_CLOCK_T) // for Illumos -#undef _CLOCK_T -#endif -static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { - #if (SIZE_MAX == UINT_MAX) - return __builtin_umul_overflow(count, size, (unsigned int *)total); - #elif (SIZE_MAX == ULONG_MAX) - return __builtin_umull_overflow(count, size, (unsigned long *)total); - #else - return __builtin_umulll_overflow(count, size, (unsigned long long *)total); - #endif -} -#else /* __builtin_umul_overflow is unavailable */ -static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { - *total = count*size; - if mi_likely(((size|count)>>(4*MI_SIZE_SIZE))==0) { // did size and count fit both in the lower half bits of a size_t? - return false; - } - else { - return (size!=0 && (SIZE_MAX / size) < count); - } -} -#endif - -// Safe multiply `count*size` into `total`; return `true` on overflow. -static inline bool mi_count_size_overflow(size_t count, size_t size, size_t* total) { - if (count==1) { // quick check for the case where count is one (common for C++ allocators) - *total = size; - return false; - } - else if mi_likely(!mi_mul_overflow(count, size, total)) { - return false; - } - else { - #if MI_DEBUG > 0 - _mi_error_message(EOVERFLOW, "allocation request is too large (%zu * %zu bytes)\n", count, size); - #endif - *total = SIZE_MAX; - return true; - } -} - - -/*---------------------------------------------------------------------------------------- - Heap functions -------------------------------------------------------------------------------------------- */ - -extern mi_decl_hidden const mi_theap_t _mi_theap_empty; // read-only empty theap, initial value of the thread local default theap (in the MI_TLS_MODEL_LOCAL) -extern mi_decl_hidden mi_theap_t _mi_theap_empty_wrong; // read-only empty theap used to signal that a theap for a heap could not be allocated - - -static inline mi_heap_t* _mi_theap_heap_peek(const mi_theap_t* theap) { - mi_assert_internal(theap!=NULL); - return mi_atomic_load_ptr_relaxed(mi_heap_t,&theap->heap); -} - -static inline mi_heap_t* _mi_theap_heap(const mi_theap_t* theap) { - mi_heap_t* const heap = _mi_theap_heap_peek(theap); - mi_assert_internal(heap!=NULL); - return heap; -} - -static inline bool mi_theap_is_initialized(const mi_theap_t* theap) { - return (theap != NULL && _mi_theap_heap_peek(theap) != NULL); -} - -static inline mi_subproc_t* _mi_theap_subproc(const mi_theap_t* theap) { - mi_subproc_t* const subproc = mi_atomic_load_ptr_relaxed(mi_subproc_t,&theap->subproc); - mi_assert_internal(!mi_theap_is_initialized(theap) || _mi_theap_heap(theap)->subproc == subproc); - return subproc; -} - -static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t xsize, bool is_wsize) { - mi_assert_internal(is_wsize ? xsize <= (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE) : xsize <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE)); - const size_t idx = (is_wsize ? xsize : mi_wsize_from_size(xsize)); - mi_assert_internal(idx < MI_PAGES_DIRECT); - return theap->pages_free_direct[idx]; -} - -static inline bool mi_theap_is_detached(mi_theap_t* theap) { - return (theap!=NULL && theap->tld->thread_id == MI_THREADID_DETACHED); -} - -static inline bool mi_theap_matches_thread(mi_theap_t* theap) { - const mi_threadid_t tid = _mi_thread_id(); - return (theap==NULL || theap->tld==NULL || theap->tld->thread_id == tid || mi_theap_is_detached(theap)); -} - -/* ----------------------------------------------------------- - The page map maps addresses to `mi_page_t` pointers ------------------------------------------------------------ */ - -#if MI_PAGE_MAP_FLAT - -// flat page-map committed on demand, using one byte per slice (64 KiB). -// single indirection and low commit, but large initial virtual reserve (4 GiB with 48 bit virtual addresses) -// used by default on <= 40 bit virtual address spaces. -extern mi_decl_hidden _Atomic(uint8_t*) _mi_page_map; -extern mi_decl_hidden _Atomic(void*) _mi_page_map_max_address; - -static inline size_t _mi_page_map_index(const void* p) { - return (size_t)((uintptr_t)p >> MI_ARENA_SLICE_SHIFT); -} - -static inline uint8_t _mi_page_map_at(size_t idx) { - return mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map)[idx]; -} - -static inline mi_page_t* _mi_ptr_page_ex(const void* p, bool* valid) { - const size_t idx = _mi_page_map_index(p); - const size_t ofs = _mi_page_map_at(idx); - if (valid != NULL) { *valid = (ofs != 0); } - return (mi_page_t*)((((uintptr_t)p >> MI_ARENA_SLICE_SHIFT) + 1 - ofs) << MI_ARENA_SLICE_SHIFT); -} - -static inline mi_page_t* _mi_checked_ptr_page(const void* p) { - #if MI_MIN_VABITS < MI_INTPTR_BITS - if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) { - if (p > mi_atomic_load_ptr_relaxed(void, &_mi_page_map_max_address)) return NULL; - } - #endif - bool valid; - mi_page_t* const page = _mi_ptr_page_ex(p, &valid); - return (valid ? page : NULL); -} - -static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) { - return _mi_ptr_page_ex(p, NULL); -} - -#else - -// 2-level page map: -// double indirection, but low commit and low virtual reserve. -// -// the page-map is usually 4 MiB (for 48 bit virtual addresses) and points to sub maps of 64 KiB. -// the page-map is committed on-demand (in 64 KiB parts) (and sub-maps are committed on-demand as well) -// one sub page-map = 64 KiB => covers 2^(16-3) * 2^16 = 2^29 = 512 MiB address space -// the page-map needs 48-(16+13) = 19 bits => 2^19 sub map pointers = 2^22 bytes = 4 MiB reserved size. -#define MI_PAGE_MAP_SUB_SHIFT (13) -#define MI_PAGE_MAP_SUB_COUNT (MI_ZU(1) << MI_PAGE_MAP_SUB_SHIFT) -#define MI_PAGE_MAP_SHIFT (MI_MAX_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT) - -typedef mi_page_t** mi_submap_t; -typedef struct mi_page_map_s { - _Atomic(size_t) committed_count; // currently committed entries - size_t reserved_size; // full reserved size (mi_page_map_t + submaps) - mi_memid_t memid; // provenance - mi_lock_t lock; // used when allocating new submaps - _Atomic(mi_submap_t) submaps[1]; -} mi_page_map_t; - -extern mi_decl_hidden _Atomic(mi_page_map_t*) __mi_page_map; - -static inline size_t _mi_page_map_index(const void* p, size_t* sub_idx) { - const size_t u = (size_t)((uintptr_t)p / MI_ARENA_SLICE_SIZE); - if (sub_idx != NULL) { *sub_idx = u % MI_PAGE_MAP_SUB_COUNT; } - return (u / MI_PAGE_MAP_SUB_COUNT); -} - -static inline mi_page_map_t* _mi_page_map(void) { - return mi_atomic_load_ptr_relaxed(mi_page_map_t,&__mi_page_map); -} - -static inline mi_submap_t _mi_page_map_at(const mi_page_map_t* pmap, size_t idx) { - return mi_atomic_load_ptr_acquire(mi_page_t*, &pmap->submaps[idx]); -} - -static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) { - const mi_page_map_t* pmap = _mi_page_map(); - size_t sub_idx; - const size_t idx = _mi_page_map_index(p, &sub_idx); - return _mi_page_map_at(pmap,idx)[sub_idx]; // NULL if p==NULL -} - -static inline mi_page_t* _mi_checked_ptr_page(const void* p) { - const mi_page_map_t* pmap = _mi_page_map(); - size_t sub_idx; - const size_t idx = _mi_page_map_index(p, &sub_idx); - const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); - if mi_unlikely(idx >= committed_count) return NULL; - // #if MI_MIN_VABITS < MI_INTPTR_BITS // is still invalid if free is called before the pagemap is initialized - // if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) { - // const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); - // if mi_unlikely(idx >= committed_count) return NULL; - // } - // #endif - mi_submap_t const sub = _mi_page_map_at(pmap,idx); - if mi_unlikely(sub == NULL) return NULL; - return sub[sub_idx]; -} - -#endif - -#if MI_PAGE_META_IS_ALIGNED -// if the page meta data is aligned in front of pages we can find it efficiently -// without needing to go through the page map (for valid pointers). -static inline mi_page_t* _mi_aligned_ptr_page0(const void* p) { - mi_page_t* const page_metas = (mi_page_t*)_mi_align_down_ptr(p,MI_PAGE_META_ALIGNMENT); - // const ptrdiff_t page_idx = ((uint8_t*)p - (uint8_t*)page_metas)/MI_ARENA_SLICE_SIZE; - const uintptr_t page_idx = ((uintptr_t)p / MI_ARENA_SLICE_SIZE) % (MI_PAGE_META_ALIGNMENT / MI_ARENA_SLICE_SIZE); - mi_assert_internal(page_idx <= MI_PAGE_META_ALIGNED_COUNT); - #if MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_RISCV // better code on x64/x86/riscv64 - mi_page_t* const page = (mi_page_t*)((uintptr_t)page_metas | (page_idx * sizeof(mi_page_t))); - #else - mi_page_t* const page = &page_metas[page_idx]; - #endif - return page; - -} - -static inline mi_page_t* _mi_aligned_ptr_page(const void* p) { - mi_page_t* const page = _mi_aligned_ptr_page0(p); - if mi_unlikely(page==NULL) return NULL; - #if MI_DEBUG - mi_page_t* const cpage = _mi_checked_ptr_page(p); - if mi_unlikely(cpage==NULL) { - _mi_error_message(EINVAL, "_mi_aligned_ptr_page: invalid pointer: %p\n", p); - return NULL; - } - #endif - return mi_atomic_load_ptr_acquire(mi_page_t, &page->self); -} -#endif - -static inline mi_page_t* _mi_ptr_page(const void* p) { - mi_assert_internal(p==NULL || mi_is_in_heap_region(p)); - #if MI_SECURE || MI_FREE_IS_CHECKED - return _mi_checked_ptr_page(p); - #elif MI_PAGE_META_IS_ALIGNED - return _mi_aligned_ptr_page(p); - #elif MI_DEBUG - return _mi_checked_ptr_page(p); - #else - return _mi_unchecked_ptr_page(p); - #endif -} - - -// Get the block size of a page -static inline size_t mi_page_block_size(const mi_page_t* page) { - mi_assert_internal(page->block_size > 0); - return page->block_size; -} - -// Page start -static inline uint8_t* mi_page_start(const mi_page_t* page) { - // multiplication must be done in `size_t` - return (uint8_t*)page + page->page_offset; -} - -static inline size_t mi_page_size(const mi_page_t* page) { - return mi_page_block_size(page) * page->reserved; -} - -static inline uint8_t* mi_page_area(const mi_page_t* page, size_t* size) { - if (size) { *size = mi_page_size(page); } - return mi_page_start(page); -} - -static inline size_t mi_page_info_size(void) { - return _mi_align_up(sizeof(mi_page_t), MI_MAX_ALIGN_SIZE); -} - -static inline bool mi_page_contains_address(const mi_page_t* page, const void* p) { - size_t psize; - uint8_t* start = mi_page_area(page, &psize); - return (start <= (uint8_t*)p && (uint8_t*)p < start + psize); -} - -static inline bool mi_page_is_in_arena(const mi_page_t* page) { - return (page->memid.memkind == MI_MEM_ARENA); -} - -static inline bool mi_page_is_singleton(const mi_page_t* page) { - return (page->reserved == 1); -} - -// Get the usable block size of a page without fixed padding. -// This may still include internal padding due to alignment and rounding up size classes. -static inline size_t mi_page_usable_block_size(const mi_page_t* page) { - return mi_page_block_size(page) - MI_PADDING_SIZE; -} - -static inline bool mi_page_meta_is_separated(const mi_page_t* page) { - #if MI_PAGE_META_IS_ALIGNED - #if MI_PAGE_META_SMALL_IS_ALIGNED - return (page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); - #else - MI_UNUSED_RELEASE(page); - mi_assert_internal(page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); - return true; - #endif - #elif MI_PAGE_META_IS_SEPARATED - // usually separated but can still be in front for direct OS allocations (due to size or alignment) or due to MI_PAGE_META_SMALL_IS_ALIGNED - return (page->memid.memkind == MI_MEM_ARENA && page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); - #else - MI_UNUSED(page); - return false; - #endif -} - -static inline uint8_t* mi_page_slice_start(const mi_page_t* page) { - if (mi_page_meta_is_separated(page)) { - // page meta info is at a separate location (at `arena->pages`) - return (uint8_t*)_mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN); - } - else { - // page meta info is at the start of the page slices - return (uint8_t*)page; - } -} - -// This gives the offset relative to the start slice of a page. -static inline size_t mi_page_slice_offset_of(const mi_page_t* page, size_t offset_relative_to_page_start) { - return (mi_page_start(page) - mi_page_slice_start(page)) + offset_relative_to_page_start; -} - -// How much of the page is committed relative to the slice start? (or 0 if fully committed already) -static inline size_t mi_page_slice_committed(const mi_page_t* page) { - return ((size_t)page->slice_pcommitted * _mi_os_page_size()); -} - -// Currently committed part of a page -static inline size_t mi_page_committed(const mi_page_t* page) { - const size_t slice_committed = mi_page_slice_committed(page); - return (slice_committed == 0 ? mi_page_size(page) : slice_committed - mi_page_slice_offset_of(page,0)); -} - -static inline size_t mi_page_used(const mi_page_t* page) { - mi_assert_internal(page != NULL); - return mi_xused_used_count(page->xused); -} - -static inline void mi_page_used_reset(mi_page_t* page) { - page->xused = mi_xused_used_reset(page->xused); -} - -static inline size_t mi_page_alloc_count(const mi_page_t* page) { - return mi_xused_alloc_count(page->xused); -} - -static inline size_t mi_page_last_used(const mi_page_t* page) { - #if MI_SIZE_SIZE >= 8 - return (page->xused.used_alloc >> 32) & 0xFFFF; - #else - return page->xlast_used; - #endif -} - -static inline size_t mi_page_last_alloc(const mi_page_t* page) { - #if MI_SIZE_SIZE >= 8 - return (page->xused.used_alloc >> 48) & 0xFFFF; - #else - return page->xlast_alloc; - #endif -} - - -// are all blocks in a page freed? -// note: needs up-to-date used count, (as the `xthread_free` list may not be empty). see `_mi_page_collect_free`. -static inline bool mi_page_all_free(const mi_page_t* page) { - mi_assert_internal(page != NULL); - return (mi_page_used(page)==0); -} - -// are there immediately available blocks, i.e. blocks available on the free list. -static inline bool mi_page_immediate_available(const mi_page_t* page) { - mi_assert_internal(page != NULL); - return (page->free != NULL); -} - - -// is the page not yet used up to its reserved space? -static inline bool mi_page_is_expandable(const mi_page_t* page) { - mi_assert_internal(page != NULL); - mi_assert_internal(page->capacity <= page->reserved); - return (page->capacity < page->reserved); -} - - -static inline bool mi_page_is_full(const mi_page_t* page) { - const bool full = (page->reserved == mi_page_used(page)); - mi_assert_internal(!full || page->free == NULL); - return full; -} - -// is more than 7/8th of a page in use? -static inline bool mi_page_is_mostly_used(const mi_page_t* page) { - if (page==NULL) return true; - uint16_t frac = page->reserved / 8U; - return (page->reserved - mi_page_used(page) <= frac); -} - -// is more than (n-1)/n'th of a page in use? -static inline bool mi_page_is_used_at_frac(const mi_page_t* page, uint16_t n) { - if (page==NULL) return true; - uint16_t frac = page->reserved / n; - return (page->reserved - mi_page_used(page) <= frac); -} - - -static inline bool mi_page_is_huge(const mi_page_t* page) { - return (mi_page_is_singleton(page) && - (page->block_size > MI_LARGE_MAX_OBJ_SIZE || - (mi_memkind_is_os(page->memid.memkind) && page->memid.mem.os.base < (void*)page))); -} - -static inline mi_page_queue_t* mi_page_queue(const mi_theap_t* theap, size_t size) { - mi_page_queue_t* const pq = &((mi_theap_t*)theap)->pages[_mi_bin(size)]; - if (size <= MI_LARGE_MAX_OBJ_SIZE) { mi_assert_internal(pq->block_size <= MI_LARGE_MAX_OBJ_SIZE); } - return pq; -} - -static inline size_t mi_page_min_commit_size(void) { - const size_t psize = _mi_os_page_size(); - return (MI_PAGE_MIN_COMMIT_SIZE >= psize ? MI_PAGE_MIN_COMMIT_SIZE : psize); -} - -//----------------------------------------------------------- -// Page thread id and flags -//----------------------------------------------------------- - -// Thread id of thread that owns this page (with flags in the bottom 2 bits) -static inline mi_threadid_t mi_page_xthread_id(const mi_page_t* page) { - return mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_id); -} - -// Plain thread id of the thread that owns this page -static inline mi_threadid_t mi_page_thread_id(const mi_page_t* page) { - return (mi_page_xthread_id(page) & ~MI_PAGE_FLAG_MASK); -} - -static inline mi_page_flags_t mi_page_flags(const mi_page_t* page) { - return (mi_page_xthread_id(page) & MI_PAGE_FLAG_MASK); -} - -static inline bool mi_page_flags_set(mi_page_t* page, bool set, mi_page_flags_t newflag) { - mi_page_flags_t old; - if (set) { old = mi_atomic_or_relaxed(&page->xthread_id, newflag); } - else { old = mi_atomic_and_relaxed(&page->xthread_id, ~newflag); } - return ((old & newflag) == newflag); -} - -static inline bool mi_page_is_in_full(const mi_page_t* page) { - return ((mi_page_flags(page) & MI_PAGE_IN_FULL_QUEUE) != 0); -} - -static inline void mi_page_set_in_full(mi_page_t* page, bool in_full) { - const bool was_in_full = mi_page_flags_set(page, in_full, MI_PAGE_IN_FULL_QUEUE); - if (was_in_full != in_full) { - // optimize: maintain pages_full_size to avoid visiting the full queue (issue #1220) - mi_theap_t* const theap = page->theap; - mi_assert_internal(theap!=NULL); - if (theap != NULL) { - mi_assert_internal(page->capacity==page->reserved); - const size_t size = page->reserved * mi_page_block_size(page); - if (in_full) { theap->pages_full_size += size; } - else { mi_assert_internal(size <= theap->pages_full_size); theap->pages_full_size -= size; } - } - } -} - -static inline bool mi_page_has_interior_pointers(const mi_page_t* page) { - return ((mi_page_flags(page) & MI_PAGE_HAS_INTERIOR_POINTERS) != 0); -} - -static inline void mi_page_set_has_interior_pointers(mi_page_t* page, bool has_aligned) { - mi_page_flags_set(page, has_aligned, MI_PAGE_HAS_INTERIOR_POINTERS); -} - -static inline void mi_page_set_theap(mi_page_t* page, mi_theap_t* theap) { - // mi_assert_internal(!mi_page_is_in_full(page)); // can happen when destroying pages on theap_destroy - page->theap = theap; - const mi_threadid_t tid = (theap == NULL ? MI_THREADID_ABANDONED : theap->tld->thread_id); - mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0); - - // we need to use an atomic cas since a concurrent thread may still set the MI_PAGE_HAS_INTERIOR_POINTERS flag (see `alloc_aligned.c`). - mi_threadid_t xtid_old = mi_page_xthread_id(page); - mi_threadid_t xtid; - do { - xtid = tid | (xtid_old & MI_PAGE_FLAG_MASK); - } while (!mi_atomic_cas_weak_release(&page->xthread_id, &xtid_old, xtid)); -} - -static inline bool mi_page_is_abandoned(const mi_page_t* page) { - // note: the xtheap field of an abandoned theap is set to the subproc (for fast reclaim-on-free) - return (mi_page_thread_id(page) <= MI_THREADID_ABANDONED_MAPPED); -} - -static inline bool mi_page_is_abandoned_mapped(const mi_page_t* page) { - return (mi_page_thread_id(page) == MI_THREADID_ABANDONED_MAPPED); -} - -static inline void mi_page_set_abandoned_mapped(mi_page_t* page) { - mi_assert_internal(mi_page_is_abandoned(page)); - mi_atomic_or_relaxed(&page->xthread_id, (mi_threadid_t)MI_THREADID_ABANDONED_MAPPED); -} - -static inline void mi_page_clear_abandoned_mapped(mi_page_t* page) { - mi_assert_internal(mi_page_is_abandoned_mapped(page)); - mi_atomic_and_relaxed(&page->xthread_id, (mi_threadid_t)MI_PAGE_FLAG_MASK); -} - - -static inline mi_theap_t* mi_page_theap(const mi_page_t* page) { - mi_assert_internal(!mi_page_is_abandoned(page)); - mi_assert_internal(page->theap != NULL && page->theap != &_mi_theap_empty); - return page->theap; -} - -static inline mi_tld_t* mi_page_tld(const mi_page_t* page) { - mi_assert_internal(!mi_page_is_abandoned(page)); - mi_assert_internal(page->theap != NULL); - return page->theap->tld; -} - - -static inline mi_heap_t* mi_page_heap(const mi_page_t* page) { - mi_heap_t* heap = page->heap; - mi_assert_internal(heap != NULL); - return heap; -} - -static inline mi_subproc_t* mi_page_subproc(const mi_page_t* page) { - mi_heap_t* const heap = mi_page_heap(page); - return heap->subproc; -} - -static inline mi_heap_t* mi_arena_heap_main(const mi_arena_t* arena) { - return _mi_subproc_heap_main(arena->subproc); -} - -static inline mi_heap_t* mi_heap_get_heap_main(const mi_heap_t* heap) { - return _mi_subproc_heap_main(heap->subproc); -} - -static inline bool _mi_is_heap_main(const mi_heap_t* heap) { - mi_assert_internal(heap!=NULL); - return (mi_heap_get_heap_main(heap) == heap); -} - -static inline bool _mi_is_process_heap_main(const mi_heap_t* heap) { - mi_assert_internal(heap!=NULL); - return (_mi_subproc_main()->heap_main == heap); -} - -//----------------------------------------------------------- -// Thread free list and ownership -//----------------------------------------------------------- - -// Thread free flag helpers -static inline mi_block_t* mi_tf_block(mi_thread_free_t tf) { - return (mi_block_t*)(tf & ~1); -} - -static inline bool mi_tf_is_owned(mi_thread_free_t tf) { - return ((tf & 1) == 1); -} - -static inline mi_thread_free_t mi_tf_create(mi_block_t* block, bool owned) { - const uintptr_t base = (uintptr_t)block | (owned ? 1 : 0); - return (mi_thread_free_t)base; -} - -static inline mi_thread_free_t mi_tf_set_owned(mi_thread_free_t tf, bool owned) { - return mi_tf_create(mi_tf_block(tf), owned); -} - -// Thread free access -static inline mi_block_t* mi_page_thread_free(const mi_page_t* page) { - return mi_tf_block(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free)); -} - -// are there any available blocks? -static inline bool mi_page_has_any_available(const mi_page_t* page) { - mi_assert_internal(page != NULL && page->reserved > 0); - return (mi_page_used(page) < page->reserved || (mi_page_thread_free(page) != NULL)); -} - -// Owned? -static inline bool mi_page_is_owned(const mi_page_t* page) { - return mi_tf_is_owned(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free)); -} - -// get ownership; returns true if the page was not owned before. -static inline bool mi_page_claim_ownership(mi_page_t* page) { - const uintptr_t old = mi_atomic_or_acq_rel(&page->xthread_free, (uintptr_t)1); - return ((old&1)==0); -} - - -/* ------------------------------------------------------------------- - Guarded and profiled objects -------------------------------------------------------------------- */ - -#define MI_SAMPLE_RATE_MAX (SIZE_MAX/4) -#define MI_SAMPLE_COUNTDOWN_MAX (MI_MAX_ALLOC_SIZE) - -static inline bool mi_theap_should_sample(mi_theap_t* theap, size_t req_size) { - // note: this should return `true` on an empty theap so we initialize it's countdown to `-1`. - mi_assert_internal(req_size <= SIZE_MAX/2); - // const size_t sample_countdown = theap->sample_countdown - req_size; - // return ((mi_ssize_t)sample_countdown < 0); - return ((mi_ssize_t)theap->sample_countdown < (mi_ssize_t)req_size); -} - -// we always align guarded pointers in a block at an offset -// the block `next` field is then used as a tag to distinguish regular offset aligned blocks from guarded ones -#define MI_BLOCK_TAG_ALIGNED ((mi_encoded_t)(0)) -#define MI_BLOCK_TAG_PROFILED ((mi_encoded_t)(1)) -#define MI_BLOCK_TAG_GUARDED (~MI_BLOCK_TAG_ALIGNED) - - -static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* p) { -#if MI_GUARDED - const ptrdiff_t offset = (uint8_t*)p - (uint8_t*)block; - return (offset >= (ptrdiff_t)(sizeof(mi_block_t)) && block->next == MI_BLOCK_TAG_GUARDED); -#else - MI_UNUSED(block); MI_UNUSED(p); - return false; -#endif -} - -static inline bool mi_block_ptr_is_sampled(const mi_block_t* block, const void* p) { -#if MI_GUARDED || MI_PROFILE - const ptrdiff_t offset = (uint8_t*)p - (uint8_t*)block; - return (offset >= (ptrdiff_t)(sizeof(mi_block_t)) && block->next != MI_BLOCK_TAG_ALIGNED); -#else - MI_UNUSED(block); MI_UNUSED(p); - return false; -#endif -} - -static inline bool mi_profiler_is_enabled(const mi_profiler_t* prof) { - _Atomic(size_t)* penabled = (_Atomic(size_t)*)&prof->reserved; - return (mi_atomic_load_acquire(penabled) != 0); -} - -static inline bool mi_profiler_set_enabled(mi_profiler_t* prof, bool enable) { - _Atomic(size_t)* penabled = (_Atomic(size_t)*)&prof->reserved; - return (mi_atomic_exchange_release(penabled, (enable ? 1 : 0)) != 0); -} - - -/* ------------------------------------------------------------------- -Encoding/Decoding the free list next pointers - -This is to protect against buffer overflow exploits where the -free list is mutated. Many hardened allocators xor the next pointer `p` -with a secret key `k1`, as `p^k1`. This prevents overwriting with known -values but might be still too weak: if the attacker can guess -the pointer `p` this can reveal `k1` (since `p^k1^p == k1`). -Moreover, if multiple blocks can be read as well, the attacker can -xor both as `(p1^k1) ^ (p2^k1) == p1^p2` which may reveal a lot -about the pointers (and subsequently `k1`). - -Instead mimalloc uses an extra key `k2` and encodes as `((p^k2)<<next, keys); - #else - MI_UNUSED(keys); MI_UNUSED(null); - next = (mi_block_t*)block->next; - #endif - mi_track_mem_noaccess(block,sizeof(mi_block_t)); - return next; -} - -static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_block_t* next, const uintptr_t* keys) { - mi_track_mem_undefined(block,sizeof(mi_block_t)); - #if MI_ENCODE_FREELIST - block->next = mi_ptr_encode(null, next, keys); - #else - MI_UNUSED(keys); MI_UNUSED(null); - block->next = (mi_encoded_t)next; - #endif - mi_track_mem_noaccess(block,sizeof(mi_block_t)); -} - -mi_block_t* _mi_block_next_is_corrupted(const mi_page_t* page, const mi_block_t* block, const mi_block_t* next); // in options.c - -static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) { - #if MI_ENCODE_FREELIST - mi_block_t* next = mi_block_nextx(page,block,page->keys); - // check for free list corruption: is `next` at least in the same page? - // todo: check if `next` is `page->block_size` aligned? - if mi_unlikely(next!=NULL && !mi_page_contains_address(page,next)) { - return _mi_block_next_is_corrupted(page,block,next); // returns NULL - } - return next; - #else - MI_UNUSED(page); - return mi_block_nextx(page,block,NULL); - #endif -} - -static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, const mi_block_t* next) { - #if MI_ENCODE_FREELIST - mi_block_set_nextx(page,block,next, page->keys); - #else - MI_UNUSED(page); - mi_block_set_nextx(page,block,next,NULL); - #endif -} - - - -/* ----------------------------------------------------------- - arena blocks ------------------------------------------------------------ */ - -// Blocks needed for a given byte size -static inline size_t mi_slice_count_of_size(size_t size) { - return _mi_divide_up(size, MI_ARENA_SLICE_SIZE); -} - -// Byte size of a number of blocks -static inline size_t mi_size_of_slices(size_t bcount) { - return (bcount * MI_ARENA_SLICE_SIZE); -} - - -/* ----------------------------------------------------------- - memory id's ------------------------------------------------------------ */ - -static inline mi_memid_t _mi_memid_create(mi_memkind_t memkind) { - mi_memid_t memid; - _mi_memzero_var(memid); - memid.memkind = memkind; - return memid; -} - -static inline mi_memid_t _mi_memid_none(void) { - return _mi_memid_create(MI_MEM_NONE); -} - -static inline mi_memid_t _mi_memid_create_os(void* base, size_t size, bool committed, bool is_zero, bool is_large) { - mi_memid_t memid = _mi_memid_create(MI_MEM_OS); - memid.mem.os.base = base; - memid.mem.os.size = size; - memid.initially_committed = committed; - memid.initially_zero = is_zero; - memid.is_pinned = is_large; - return memid; -} - -static inline mi_memid_t _mi_memid_create_static(void* p, size_t size) { - mi_memid_t memid = _mi_memid_create(MI_MEM_STATIC); - memid.mem.malloc.base = p; - memid.mem.malloc.size = size; - memid.initially_committed = true; - memid.is_pinned = true; - return memid; -} - -static inline mi_memid_t _mi_memid_create_malloc(void* p, size_t size, bool iszero) { - mi_memid_t memid = _mi_memid_create(MI_MEM_MALLOC); - memid.mem.malloc.base = p; - memid.mem.malloc.size = size; - memid.initially_committed = true; - memid.initially_zero = iszero; - memid.is_pinned = true; - return memid; -} - -static inline size_t _mi_memid_size(mi_memid_t memid) { - if (mi_memid_is_os(memid)) { - return memid.mem.os.size; - } - else if (memid.memkind == MI_MEM_ARENA) { - return mi_size_of_slices(memid.mem.arena.slice_count); - } - else if (memid.memkind == MI_MEM_MALLOC) { - return memid.mem.malloc.size; - } - else { - mi_assert_internal(mi_memid_needs_no_free(memid)); - return 0; - } -} - -// ------------------------------------------------------------------- -// Fast "random" shuffle -// ------------------------------------------------------------------- - -static inline size_t _mi_random_shuffle(size_t x) { - if (x==0) { x = 17; } // ensure we don't get stuck in generating zeros -#if (MI_SIZE_SIZE>=8) - // by Sebastiano Vigna, see: - x ^= x >> 30; - x *= 0xbf58476d1ce4e5b9UL; - x ^= x >> 27; - x *= 0x94d049bb133111ebUL; - x ^= x >> 31; -#elif (MI_SIZE_SIZE==4) - // by Chris Wellons, see: - x ^= x >> 16; - x *= 0x7feb352dUL; - x ^= x >> 15; - x *= 0x846ca68bUL; - x ^= x >> 16; -#endif - return x; -} - - -// --------------------------------------------------------------------------------- -// Provide our own `_mi_memcpy/set` for potential performance optimizations. -// --------------------------------------------------------------------------------- - -static inline void* _mi_memcpy(void* dst, const void* src, size_t n) { - return memcpy(dst, src, n); -} - -static inline void* _mi_memset(void* dst, int val, size_t n) { - return memset(dst, val, n); -} - -static inline void* _mi_memset_backward(void* dst, int val, size_t n) { - memset((uint8_t*)dst - n, val, n); - return dst; -} - -static inline void* _mi_memzero(void* dst, size_t n) { - return _mi_memset(dst, 0, n); -} - -static inline void* _mi_memzero_backward(void* dst, size_t n) { - return _mi_memset_backward(dst, 0, n); -} +#include "util/libc.h" +#include "util/options.h" +#include "util/random.h" +#include "util/stats.h" +#include "util/threadlocal.h" -static inline void* _mi_memcpy_aligned(void* dst, const void* src, size_t n) { - // on gcc/clang we can provide a hint that the pointers are word aligned. - mi_assert_internal(_mi_is_aligned(dst,MI_SIZE_SIZE) && _mi_is_aligned(src,MI_SIZE_SIZE)); - void* adst = mi_assume_aligned(dst, MI_SIZE_SIZE); - const void* asrc = mi_assume_aligned(src, MI_SIZE_SIZE); - return _mi_memcpy(adst, asrc, n); -} +#include "arena/arena.h" +#include "arena/arena-alloc.h" +#include "arena/arena-page.h" -static inline void* _mi_memset_aligned(void* dst, int val, size_t n) { - mi_assert_internal(_mi_is_aligned(dst,MI_SIZE_SIZE)); - void* adst = mi_assume_aligned(dst, MI_SIZE_SIZE); - return _mi_memset(adst, val, n); -} +#include "heap/page-map.h" +#include "heap/subproc.h" +#include "heap/init.h" +#include "heap/theap.h" +#include "heap/heap.h" -static inline void* _mi_memzero_aligned(void* dst, size_t n) { - return _mi_memset_aligned(dst, 0, n); -} +#include "alloc/page.h" +#include "alloc/alloc.h" +#include "alloc/free.h" -// Zero a block: blocks are always aligned with a positive bsize in machine-word bytes. -static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize) { - mi_assert_internal(bsize%MI_SIZE_SIZE == 0); - mi_assert_internal(bsize > 0); - mi_assert_internal(_mi_is_aligned(dst,MI_SIZE_SIZE)); - mi_assert_internal(bsize < MI_MAX_ALIGN_SIZE || _mi_is_aligned(dst,MI_MAX_ALIGN_SIZE)); - - // fast memzero for small sizes based on overlapping writes (and assuming non-zero size_t-multiple size, and size_t aligned) - // assumes constant memset(p,0,N) gets optimized to fast simd stores by the compiler - // note: disabled on riscv for now as a constant memset is not always replaced correctly by current compilers. - // (compile with -DMI_USE_MEMZERO16X=0 to disable this) - #if (!defined(MI_USE_MEMZERO16X) && !MI_ARCH_RISCV) || (MI_USE_MEMZERO16X != 0) // 16x MI_SIZE_SIZE (128 bytes on 64-bit) - if mi_unlikely(bsize < 2*MI_SIZE_SIZE) { // bsize < 16 (8) - *((size_t*)dst) = 0; - return dst; - } - mi_assert_internal(_mi_is_aligned(dst,MI_MAX_ALIGN_SIZE)); - uint8_t* const start = (uint8_t*)mi_assume_aligned(dst, MI_MAX_ALIGN_SIZE); - uint8_t* const end = start + bsize; // note: if bsize is always a multiple of 16 then end is always aligned as well (but due to padding this does not hold) - if mi_likely(bsize < 8*MI_SIZE_SIZE) { // bsize < 64 (32) - const size_t ofs = (bsize>>1)&(2*MI_SIZE_SIZE); mi_assert_internal(bsize < 4*MI_SIZE_SIZE ? ofs==0 : ofs==2*MI_SIZE_SIZE); // ofs == 16 (8) - _mi_memzero(start, 2*MI_SIZE_SIZE); - _mi_memzero(start+ofs, 2*MI_SIZE_SIZE); - _mi_memzero_backward(end-ofs, 2*MI_SIZE_SIZE); - _mi_memzero_backward(end, 2*MI_SIZE_SIZE); - return dst; - } - if mi_likely(bsize <= 16*MI_SIZE_SIZE) { // bsize < 128 (64) - _mi_memzero(start, 8*MI_SIZE_SIZE); - _mi_memzero_backward(end, 8*MI_SIZE_SIZE); - return dst; - } - #endif - // fallback to regular memset for larger sizes - void* const wdst = mi_assume_aligned(dst,MI_SIZE_SIZE); - return _mi_memzero_aligned(wdst, bsize); -} +#include "profile/sample-profile.h" +#include "profile/sample-guarded.h" +#include "profile/pprof.h" #endif // MI_INTERNAL_H diff --git a/system/lib/mimalloc/include/mimalloc/prim.h b/system/lib/mimalloc/include/mimalloc/prim.h index 82b2d9c3d242e..38e9499054b5f 100644 --- a/system/lib/mimalloc/include/mimalloc/prim.h +++ b/system/lib/mimalloc/include/mimalloc/prim.h @@ -24,7 +24,7 @@ terms of the MIT license. A copy of the license can be found in the file // OS memory configuration typedef struct mi_os_mem_config_s { size_t page_size; // default to 4KiB - size_t large_page_size; // 0 if not supported, usually 2MiB + size_t large_page_size; // 0 if not supported, usually 2MiB size_t alloc_granularity; // smallest allocation size (usually 4KiB, on Windows 64KiB) size_t physical_memory_in_kib; // physical memory size in KiB size_t virtual_address_bits; // usually 48 or 56 bits on 64-bit systems. (used to determine secure randomization) @@ -133,4 +133,22 @@ bool _mi_prim_thread_is_in_threadpool(void); // Is called only in rare situations and does not have to be lightning fast. void _mi_prim_thread_yield(void); +// Return `true` if the process is forcefully shutdown (with potentially killed threads that might still hold a lock) +// If this returns true, mimalloc skips graceful shutdown (as locks might still be held by threads that are killed). +// See also issue #1377 +bool _mi_prim_process_is_killed(void); + + +// The following are predefined by default + +// Is the mimalloc-redirect running (Windows only) +bool _mi_is_redirected(void); + +// Called to initialize the allocator +bool _mi_allocator_init(const char** message); + +// Called to clean up the allocator +void _mi_allocator_done(void); + + #endif // MI_PRIM_H diff --git a/system/lib/mimalloc/include/mimalloc/prim/os.h b/system/lib/mimalloc/include/mimalloc/prim/os.h new file mode 100644 index 0000000000000..a2ac2d4fd9383 --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/prim/os.h @@ -0,0 +1,56 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_INTERNAL_OS_H +#define MI_INTERNAL_OS_H + +#include "../types.h" + +void _mi_os_init(void); // called from process init +void* _mi_os_alloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid); +void* _mi_os_zalloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid); +void _mi_os_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid); +void _mi_os_free_ex(mi_subproc_t* subproc, void* p, size_t size, bool still_committed, mi_memid_t memid ); + +size_t _mi_os_page_size(void); +size_t _mi_os_guard_page_size(void); +size_t _mi_os_good_alloc_size(size_t size); +bool _mi_os_has_overcommit(void); +bool _mi_os_has_virtual_reserve(void); +bool _mi_os_has_thp(void); +size_t _mi_os_virtual_address_bits(void); +size_t _mi_os_minimal_purge_size(void); + +bool _mi_os_reset(mi_subproc_t* subproc, void* addr, size_t size); +bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size); +void _mi_os_reuse(mi_subproc_t* subproc, void* p, size_t size); +mi_decl_nodiscard bool _mi_os_commit(mi_subproc_t* subproc, void* p, size_t size, bool* is_zero); +mi_decl_nodiscard bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero, size_t stat_already_committed); +mi_decl_nodiscard bool _mi_os_protect(void* addr, size_t size); +bool _mi_os_unprotect(void* addr, size_t size); +bool _mi_os_purge(mi_subproc_t* subproc, void* p, size_t size); +bool _mi_os_purge_ex(mi_subproc_t* subproc, void* p, size_t size, bool allow_reset, size_t stats_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg); + +size_t _mi_os_secure_guard_page_size(void); +bool _mi_os_secure_guard_page_set_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid); +bool _mi_os_secure_guard_page_set_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid); +bool _mi_os_secure_guard_page_reset_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid); +bool _mi_os_secure_guard_page_reset_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid); + +int _mi_os_numa_node(void); +int _mi_os_numa_node_count(void); + +void* _mi_os_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid); +void* _mi_os_alloc_aligned_at_offset(mi_subproc_t* subproc, size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid); + +void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size); +bool _mi_os_canuse_large_page(size_t size, size_t alignment); + +size_t _mi_os_large_page_size(void); +void* _mi_os_alloc_huge_os_pages(mi_subproc_t* subproc, size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid); + +#endif diff --git a/system/lib/mimalloc/include/mimalloc/prim-tls.h b/system/lib/mimalloc/include/mimalloc/prim/prim-tls.h similarity index 91% rename from system/lib/mimalloc/include/mimalloc/prim-tls.h rename to system/lib/mimalloc/include/mimalloc/prim/prim-tls.h index da78d0193dff9..110c831dba973 100644 --- a/system/lib/mimalloc/include/mimalloc/prim-tls.h +++ b/system/lib/mimalloc/include/mimalloc/prim/prim-tls.h @@ -5,15 +5,17 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ #pragma once -#ifndef MIMALLOC_PRIM_TLS_H -#define MIMALLOC_PRIM_TLS_H +#ifndef MI_INTERNAL_PRIM_TLS_H +#define MI_INTERNAL_PRIM_TLS_H -#include "types.h" -#include "internal.h" // mi_decl_hidden +#include "../types.h" +#include "../heap/heap.h" +#include "../heap/theap.h" +#include "../util/threadlocal.h" // -------------------------------------------------------------------------- -// We need fast access to both a unique thread id (in `free.c:mi_free`) and -// to a thread-local theap pointer (in `alloc.c:mi_malloc`). +// We need fast access to both a unique thread id (in `alloc/free.c:mi_free`) and +// to a thread-local theap pointer (in `alloc/alloc.c:mi_malloc`). // // For performance, we tend to use specialized code for various platforms. // This leads to quite a few ifdefs but it is just for performance and there @@ -25,10 +27,6 @@ terms of the MIT license. A copy of the license can be found in the file // Android,OpenBSD : use pthread locals (MI_TLS_MODEL_PTHREADS). todo: maybe on Android MI_TLS_MODEL_LOCAL is better? // -------------------------------------------------------------------------- -// static inline void** mi_prim_thread_pointer(void); // get a pointer to the thread local storage -// static inline void* mi_prim_tls_slot(size_t slot); // directly read an entry from the thread local storage block -// static inline void mi_prim_tls_slot_set(size_t slot, void* value); - static inline mi_threadid_t _mi_prim_thread_id(void); // get a unique id for a thread static inline mi_theap_t* _mi_theap_default(void); // the default thread local theap static inline mi_theap_t* _mi_theap_cached(void); // last used thread local theap using the _heap_ api @@ -37,6 +35,19 @@ static inline mi_theap_t* _mi_heap_theap(mi_heap_t* heap); // static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap); // get the theap but don't update _mi_theap_cached static inline mi_theap_t* _mi_page_associated_theap_peek(const mi_page_t* page); // get the theap associated with a page (used in `mi_free_collect_mt`) +// Regular exports +void _mi_tls_slots_init(void); +void _mi_tls_slots_done(void); +mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; +void _mi_theap_default_set(mi_theap_t* theap); +void _mi_theap_cached_set(mi_theap_t* theap); + + +static inline bool mi_theap_matches_thread(mi_theap_t* theap) { + const mi_threadid_t tid = _mi_thread_id(); + return (theap==NULL || theap->tld==NULL || theap->tld->thread_id == tid || mi_theap_is_detached(theap)); +} + // Default TLS model #if !defined(MI_TLS_MODEL_LOCAL) && !defined(MI_TLS_MODEL_PTHREADS) && !defined(MI_TLS_MODEL_FIXED) && !defined(MI_TLS_MODEL_WIN32) @@ -87,13 +98,14 @@ static inline void** mi_prim_thread_pointer(void) { #elif defined(__GNUC__) && !defined(__CYGWIN__) #if defined(__aarch64__) static inline void** mi_prim_thread_pointer(void) { - void** tcb; + // Use a 64-bit temporary: on arm64_32 (ILP32), pointers are 32 bits but system registers are 64 bits. + uint64_t tp; #if defined(__APPLE__) // M1, issue rgb(62, 76, 62) - __asm__ ("mrs %0, tpidrro_el0\n\tbic %0, %0, #7" : "=r" (tcb)); + __asm__ ("mrs %0, tpidrro_el0\n\tbic %0, %0, #7" : "=r" (tp)); #else - __asm__ ("mrs %0, tpidr_el0" : "=r" (tcb)); + __asm__ ("mrs %0, tpidr_el0" : "=r" (tp)); #endif - return tcb; + return (void**)((uintptr_t)tp); } #elif defined(__riscv) static inline void** mi_prim_thread_pointer(void) { @@ -249,7 +261,7 @@ static inline mi_theap_t* __mi_theap_empty(void) { extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default; // default theap to allocate from extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached; // theap from the last used heap -// defined in `init.c`; do not use these directly +// defined in `heap/init.c`; do not use these directly extern mi_decl_hidden bool _mi_process_is_initialized; // has mi_process_init been called? static inline mi_theap_t* _mi_theap_default(void) { @@ -276,7 +288,7 @@ static inline mi_theap_t* _mi_theap_default(void) { pthread_key_t key = mi_atomic_load_relaxed(&_mi_theap_default_key); #if defined(__APPLE__) && defined(__aarch64__) && MI_HAS_TLS_SLOT // on apple arm64, the pthread specific slots are direct slots; inline it to avoid a stack frame setup in `mi_malloc` - // todo: this is probably also the case on x64 and power pc? + // todo: this is probably also the case on x64 and power pc? if (key == MI_PTHREAD_KEY_INVALID) return NULL; return (mi_theap_t*)mi_prim_tls_slot(key); #else @@ -389,9 +401,9 @@ static inline bool _mi_thread_is_initialized(void) { static inline mi_theap_t* _mi_heap_theap(mi_heap_t* heap) { mi_theap_t* theap = _mi_theap_cached(); #if MI_THEAP_INITASNULL - if mi_likely(theap!=NULL && _mi_theap_heap_peek(theap)==heap) return theap; + if mi_likely(theap!=NULL && mi_theap_heap_peek(theap)==heap) return theap; #else - if mi_likely(_mi_theap_heap_peek(theap)==heap) return theap; + if mi_likely(mi_theap_heap_peek(theap)==heap) return theap; #endif return _mi_heap_theap_get_or_init(heap); } @@ -399,9 +411,9 @@ static inline mi_theap_t* _mi_heap_theap(mi_heap_t* heap) { static inline mi_theap_t* _mi_heap_theap_cached(mi_heap_t* heap) { mi_theap_t* theap = _mi_theap_cached(); #if MI_THEAP_INITASNULL - if mi_likely(theap!=NULL && _mi_theap_heap_peek(theap)==heap) return theap; + if mi_likely(theap!=NULL && mi_theap_heap_peek(theap)==heap) return theap; #else - if mi_likely(_mi_theap_heap_peek(theap)==heap) return theap; + if mi_likely(mi_theap_heap_peek(theap)==heap) return theap; #endif return NULL; } @@ -410,9 +422,9 @@ static inline mi_theap_t* _mi_heap_theap_cached(mi_heap_t* heap) { static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap) { mi_theap_t* theap = _mi_theap_cached(); #if MI_THEAP_INITASNULL - if mi_likely(theap!=NULL && _mi_theap_heap_peek(theap)==heap) return theap; + if mi_likely(theap!=NULL && mi_theap_heap_peek(theap)==heap) return theap; #else - if mi_likely(_mi_theap_heap_peek(theap)==heap) return theap; + if mi_likely(mi_theap_heap_peek(theap)==heap) return theap; #endif theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); // don't update the cache on a query mi_assert_internal(theap==NULL || (!_mi_is_empty_theap(theap) && theap->heap==heap)); @@ -420,9 +432,9 @@ static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap) { } // Find the associated theap or NULL if it does not exist (during shutdown) -// Should be fast as it is called in `free.c:mi_free_try_collect`. +// Should be fast as it is called in `alloc/free.c:mi_free_try_collect`. static inline mi_theap_t* _mi_page_associated_theap_peek(const mi_page_t* page) { - mi_heap_t* const heap = mi_page_heap(page); + mi_heap_t* const heap = page->heap; mi_theap_t* const theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); if (theap==NULL || theap->tld==NULL /* heap destroy */) return NULL; if (theap->heap != heap) return NULL; // should never happen, but can happen for a free across subprocesses, which can happen during pthread tls storage deallocation diff --git a/system/lib/mimalloc/include/mimalloc/profile/pprof.h b/system/lib/mimalloc/include/mimalloc/profile/pprof.h new file mode 100644 index 0000000000000..c2c641401e7a1 --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/profile/pprof.h @@ -0,0 +1,16 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_INTERNAL_PPROF_H +#define MI_INTERNAL_PPROF_H + +#include "../types.h" + +void _mi_pprof_profiler_init(void); +void _mi_pprof_profiler_done(void); + +#endif diff --git a/system/lib/mimalloc/include/mimalloc/profile/sample-guarded.h b/system/lib/mimalloc/include/mimalloc/profile/sample-guarded.h new file mode 100644 index 0000000000000..4e8cd06272bfb --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/profile/sample-guarded.h @@ -0,0 +1,31 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_INTERNAL_SAMPLE_GUARDED_H +#define MI_INTERNAL_SAMPLE_GUARDED_H + +#include "../types.h" +#include "sample-profile.h" + +void _mi_page_unguard_all(mi_page_t* page); +mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; +void _mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p); +void _mi_theap_guarded_init(mi_theap_t* theap); + +#define MI_BLOCK_TAG_GUARDED (~MI_BLOCK_TAG_ALIGNED) + +static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* p) { +#if MI_GUARDED + const ptrdiff_t offset = (uint8_t*)p - (uint8_t*)block; + return (offset >= (ptrdiff_t)(sizeof(mi_block_t)) && block->next == MI_BLOCK_TAG_GUARDED); +#else + MI_UNUSED(block); MI_UNUSED(p); + return false; +#endif +} + +#endif diff --git a/system/lib/mimalloc/include/mimalloc/profile/sample-profile.h b/system/lib/mimalloc/include/mimalloc/profile/sample-profile.h new file mode 100644 index 0000000000000..645089d0ae543 --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/profile/sample-profile.h @@ -0,0 +1,70 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_INTERNAL_SAMPLE_PROFILE_H +#define MI_INTERNAL_SAMPLE_PROFILE_H + +#include "../types.h" + +mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage) mi_attr_noexcept; +size_t _mi_theap_update_sample_rate(mi_theap_t* theap); + +mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, uint64_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept; +void _mi_page_profile_on_free(mi_page_t* page, mi_block_t* block, void* p); +size_t _mi_theap_set_profile_sample_rate(mi_theap_t* theap, size_t sample_rate); + +// permanently exclude a theap from profiling (mirrors `mi_heap_profile_disable` but at the theap level); +// used for detached/meta theaps used to bootstrap thread/theap metadata, since sampling those can call +// back into the profiler while allocating on a not yet (re-)initialized thread, causing deadlock. +static inline void mi_theap_profile_never(mi_theap_t* theap) { + theap->profile_never = true; + theap->profile_sample_rate = 0; + theap->profile_sample_countdown = 0; +} + +/* ------------------------------------------------------------------- + Guarded and profiled objects +------------------------------------------------------------------- */ + +#define MI_SAMPLE_RATE_MAX (SIZE_MAX/4) +#define MI_SAMPLE_COUNTDOWN_MAX (MI_MAX_ALLOC_SIZE) + +static inline bool mi_theap_should_sample(mi_theap_t* theap, size_t req_size) { + // note: this should return `true` on an empty theap so we initialize it's countdown to `-1`. + mi_assert_internal(req_size <= SIZE_MAX/2); + // const size_t sample_countdown = theap->sample_countdown - req_size; + // return ((mi_ssize_t)sample_countdown < 0); + return ((mi_ssize_t)theap->sample_countdown < (mi_ssize_t)req_size); +} + +// we always align guarded pointers in a block at an offset +// the block `next` field is then used as a tag to distinguish regular offset aligned blocks from guarded ones +#define MI_BLOCK_TAG_ALIGNED ((mi_encoded_t)(0)) +#define MI_BLOCK_TAG_PROFILED ((mi_encoded_t)(1)) + + +static inline bool mi_block_ptr_is_sampled(const mi_block_t* block, const void* p) { +#if MI_GUARDED || MI_PROFILE + const ptrdiff_t offset = (uint8_t*)p - (uint8_t*)block; + return (offset >= (ptrdiff_t)(sizeof(mi_block_t)) && block->next != MI_BLOCK_TAG_ALIGNED); +#else + MI_UNUSED(block); MI_UNUSED(p); + return false; +#endif +} + +static inline bool mi_profiler_is_enabled(const mi_profiler_t* prof) { + _Atomic(size_t)* penabled = (_Atomic(size_t)*)&prof->reserved; + return (mi_atomic_load_acquire(penabled) != 0); +} + +static inline bool mi_profiler_set_enabled(mi_profiler_t* prof, bool enable) { + _Atomic(size_t)* penabled = (_Atomic(size_t)*)&prof->reserved; + return (mi_atomic_exchange_release(penabled, (enable ? 1 : 0)) != 0); +} + +#endif diff --git a/system/lib/mimalloc/include/mimalloc/qsbr.h b/system/lib/mimalloc/include/mimalloc/qsbr.h new file mode 100644 index 0000000000000..a91e8ad98dde6 --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/qsbr.h @@ -0,0 +1,56 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +/* ---------------------------------------------------------------------------- + Quiescent State-Based Reclamation (qsbr). + QSBR as used in CPython is a technique for safe memory reclamation in concurrent + environments. In such case we need to delay free-ing pages until there are no + more concurrent readers; the interface below provides the necessary primitives + for managing QSBR state so mimalloc can delay freeing pages until it is + safe to do so (and efficiently reuse them as well). + + See `/test/test-qsbr.c` for a minimal example implementation. +-----------------------------------------------------------------------------*/ + +#pragma once +#ifndef MI_QSBR_H +#define MI_QSBR_H + +#include +#include + +#ifndef MI_QSBR +#if defined(MI_CPYTHON) +#define MI_QSBR 1 +#else +#define MI_QSBR 0 +#endif +#endif + +// Create a qsbr enabled heap +mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_qsbr(void); + +#if MI_QSBR +// Define these for the host runtime to implement qsbr bookkeeping. +typedef struct mi_page_qsbr_s mi_page_qsbr_t; +typedef struct mi_heap_qsbr_s mi_heap_qsbr_t; + +// Host-defined runtime bookkeeping primitives; see doc/qsbr.md. (called by mimalloc for pages in qsbr heaps) +#ifdef __cplusplus +extern "C" { +#endif +void mi_heap_qsbr_init(mi_heap_qsbr_t* heap); +void mi_page_qsbr_clear(mi_page_qsbr_t* page); +bool mi_page_qsbr_is_clear(const mi_page_qsbr_t* page); +// Establish a goal if needed, then check whether release of the empty page must be delayed. +bool mi_page_qsbr_should_delay(mi_page_qsbr_t* page, mi_heap_qsbr_t* heap, size_t page_bytes); +#ifdef __cplusplus +} +#endif +#endif + +#endif diff --git a/system/lib/mimalloc/include/mimalloc/readme.md b/system/lib/mimalloc/include/mimalloc/readme.md new file mode 100644 index 0000000000000..1fa5832ef8338 --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/readme.md @@ -0,0 +1,21 @@ +# Internal Headers + +- [`types.h`](types.h): Defines the types of all internal mimalloc datastructures +and configuration options. +- [`decl.h`](decl.h): Compiler attributes, assertions, initialization macros, and platform-specific declarations. +- [`bits.h`](bits.h): Architecture dependent type definitions, and bit operations. +- [`atomic.h`](atomic.h): Portable atomic operations and locks. +- [`prim.h`](prim.h): Primitive and minimal portability API needed from a host system. +- [`track.h`](track.h): Memory-tracking integration for Valgrind, AddressSanitizer, and Windows event tracing. +- [`qsbr.h`](qsbr.h): Quiescent-state based reclamation primitives (for runtime systems like CPython). +- [`internal.h`](internal.h): Deprecated: umbrella header for internal allocator declarations and utilities. + +# Source Headers + +- [`alloc`](alloc): Internal allocation, freeing, and page-management interfaces. +- [`arena`](arena): Arena management, slice allocation, and arena-backed page interfaces. +- [`heap`](heap): Heap, thread-local heap, subprocess, initialization, and page-map interfaces. +- [`prim`](prim): Higher-level OS memory operations and primitive thread-local access. +- [`profile`](profile): Sampled allocation, guarded allocation, and profiler interfaces. +- [`util`](util): Runtime utilities, options, randomness, statistics, and dynamic thread-local interfaces. + diff --git a/system/lib/mimalloc/include/mimalloc/track.h b/system/lib/mimalloc/include/mimalloc/track.h index e6a57c8ec4894..f2a24aacfe617 100644 --- a/system/lib/mimalloc/include/mimalloc/track.h +++ b/system/lib/mimalloc/include/mimalloc/track.h @@ -89,6 +89,19 @@ defined, undefined, or not accessible at all: #define mi_track_malloc_size(p,reqsize,size,zero) EventWriteETW_MI_ALLOC((UINT64)(p), size) #define mi_track_free_size(p,size) EventWriteETW_MI_FREE((UINT64)(p), size) +#elif defined(MI_TRACK_CUSTOM) +// application-defined hooks + +#define MI_TRACK_ENABLED 0 +#define MI_TRACK_HEAP_DESTROY 0 +#define MI_TRACK_TOOL "custom" + +void mi_track_malloc_hook(const void* p, size_t reqsize, size_t size, bool zero) mi_attr_noexcept; +void mi_track_free_hook(const void* p, size_t size) mi_attr_noexcept; + +#define mi_track_malloc_size(p,reqsize,size,zero) mi_track_malloc_hook((p),(reqsize),(size),(zero)) +#define mi_track_free_size(p,size) mi_track_free_hook((p),(size)) + #else // no tracking diff --git a/system/lib/mimalloc/include/mimalloc/types.h b/system/lib/mimalloc/include/mimalloc/types.h index 530c464a36876..ebdff4e876081 100644 --- a/system/lib/mimalloc/include/mimalloc/types.h +++ b/system/lib/mimalloc/include/mimalloc/types.h @@ -22,16 +22,19 @@ terms of the MIT license. A copy of the license can be found in the file // mi_subproc_t : all heaps belong to a sub-process (usually just the main one) // -------------------------------------------------------------------------- - -#include -#include +#include "../mimalloc.h" +#include "../mimalloc-stats.h" +#include "../mimalloc-profile.h" #include // ptrdiff_t #include // uintptr_t, uint16_t, etc #include // bool #include // SIZE_MAX etc. #include // error codes + +#include "decl.h" // compiler dependent definitions #include "bits.h" // size defines (MI_SIZE_SIZE etc), bit operations #include "atomic.h" // _Atomic primitives +#include "qsbr.h" // quiescent state-based reclamation (qsbr) primitives (for runtime systems like CPython) // Minimal alignment necessary. On most platforms 16 bytes are needed // due to SSE registers for example. This must be at least `sizeof(void*)` @@ -54,6 +57,7 @@ terms of the MIT license. A copy of the license can be found in the file // #define MI_TRACK_VALGRIND 1 // #define MI_TRACK_ASAN 1 // #define MI_TRACK_ETW 1 +// #define MI_TRACK_CUSTOM 1 // Define MI_STATS as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance). // #define MI_STATS 1 @@ -61,7 +65,7 @@ terms of the MIT license. A copy of the license can be found in the file // Define MI_SECURE to enable security mitigations // #define MI_SECURE 1 // check invalid pointer free, guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption // #define MI_SECURE 2 // randomize relative allocation addresses within mimalloc pages -// #define MI_SECURE 3 // check buffer overflow, check double free, encode free lists (and detect corrupted free lists) +// #define MI_SECURE 3 // check buffer overflow, check double free, encode free lists (and detect corrupted free lists) // #define MI_SECURE 4 // same as level 3 for now (`-DMI_SECURE=ON`) // #define MI_SECURE 5 // guard page at the end of each mimalloc page (expensive!) (`-DMI_SECURE_FULL=ON`), and byte-precise buffer overflow checks. @@ -113,7 +117,7 @@ terms of the MIT license. A copy of the license can be found in the file // Reserve extra padding at the end of each block to be more resilient against theap block overflows. // The padding can detect heap-block overflow on free, and provides byte-precise `mi_usable_size`. -#if !defined(MI_PADDING) +#if !defined(MI_PADDING) #if (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW)) #define MI_PADDING 1 #else @@ -168,7 +172,7 @@ terms of the MIT license. A copy of the license can be found in the file // free `mi_cfree` can still use the page map to validate pointers. #if !MI_FREE_IS_CHECKED && !MI_FREE_USE_PAGEMAP #if MI_PAGE_META_IS_SEPARATED -#define MI_PAGE_META_IS_ALIGNED 1 +#define MI_PAGE_META_IS_ALIGNED 1 #define MI_PAGE_META_ALIGNED_CHUNKS MI_SIZE_SIZE #else #warning "cannot optimize free with alignment since the page meta data is not separated (due to MI_PAGE_MAP_FLAT?)" @@ -198,9 +202,7 @@ terms of the MIT license. A copy of the license can be found in the file #if MI_PAGE_META_IS_SEPARATED && MI_PAGE_MAP_FLAT #error "cannot have a flat page map with separated page infos" #endif -#if MI_DEBUG && NDEBUG -#warning "mimalloc assertions enabled in a release build" -#endif + // -------------------------------------------------------------- // Sizes of internal data-structures @@ -211,9 +213,14 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MI_ARENA_SLICE_SHIFT #ifdef MI_SMALL_PAGE_SHIFT // backward compatibility #define MI_ARENA_SLICE_SHIFT MI_SMALL_PAGE_SHIFT - #elif MI_SECURE>=5 && ((__APPLE__ && MI_ARCH_ARM64) || (defined(PAGE_SIZE) && PAGE_SIZE >= 16*MI_KiB)) + #elif MI_SECURE>=5 && __APPLE__ && MI_ARCH_ARM64 #define MI_ARENA_SLICE_SHIFT (17) // 128 KiB to not waste too much due to 16 KiB guard pages - #else + #elif MI_SECURE>=5 && defined(PAGE_SIZE) && !defined(__wasi__) // see issue #1440 + #if PAGE_SIZE >= 16*MI_KiB + #define MI_ARENA_SLICE_SHIFT (17) // 128 KiB to not waste too much due to 16 KiB guard pages + #endif + #endif + #ifndef MI_ARENA_SLICE_SHIFT #define MI_ARENA_SLICE_SHIFT (13 + MI_SIZE_SHIFT) // 64 KiB (32 KiB on 32-bit) #endif #endif @@ -234,7 +241,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_BCHUNK_BITS (1 << MI_BCHUNK_BITS_SHIFT) // sub-bitmaps in arena's are "bchunks" of 512 bits #define MI_ARENA_SLICE_SIZE (MI_ZU(1) << MI_ARENA_SLICE_SHIFT) // arena's allocate in slices of 64 KiB #define MI_ARENA_SLICE_ALIGN (MI_ARENA_SLICE_SIZE) -#define MI_ARENA_CHUNK_SIZE (MI_BCHUNK_BITS * MI_ARENA_SLICE_SIZE) +#define MI_ARENA_CHUNK_SIZE (MI_BCHUNK_BITS * MI_ARENA_SLICE_SIZE) #define MI_ARENA_MIN_OBJ_SLICES (1) #define MI_ARENA_MAX_CHUNK_OBJ_SLICES (MI_BCHUNK_BITS) // 32 MiB (or 8 MiB on 32-bit) @@ -261,7 +268,7 @@ terms of the MIT license. A copy of the license can be found in the file // We never allocate more than PTRDIFF_MAX (see also ) #define MI_MAX_ALLOC_SIZE PTRDIFF_MAX -// Minimal commit for a page on-demand commit +// Minimal commit for a page on-demand commit #define MI_PAGE_MIN_COMMIT_SIZE (16*MI_KiB) /* MI_ARENA_SLICE_SIZE */ #if MI_PAGE_META_IS_ALIGNED @@ -310,11 +317,11 @@ typedef struct mi_subproc_s mi_subproc_t; typedef enum mi_memkind_e { MI_MEM_NONE, // not allocated (or static) MI_MEM_EXTERNAL, // not owned by mimalloc but provided externally (via `mi_manage_os_memory` for example) - MI_MEM_STATIC, // allocated in a static area and should not be freed (the initial main theap data for example (`init.c`)) + MI_MEM_STATIC, // allocated in a static area and should not be freed (the initial main theap data for example (`heap/init.c`)) MI_MEM_OS, // allocated from the OS MI_MEM_OS_HUGE, // allocated as huge OS pages (usually 1GiB, pinned to physical memory) MI_MEM_OS_REMAP, // allocated in a remapable area (i.e. using `mremap`) - MI_MEM_ARENA, // allocated from an arena (the usual case) (`arena.c`) + MI_MEM_ARENA, // allocated from an arena (the usual case) (`arena/arena-alloc.c`) MI_MEM_MALLOC // allocated with mi_malloc } mi_memkind_t; @@ -372,6 +379,14 @@ static inline mi_arena_t* mi_memid_arena(mi_memid_t memid) { return (memid.memkind == MI_MEM_ARENA ? memid.mem.arena.arena : NULL); } +// defined in arena.c +mi_memid_t _mi_memid_create(mi_memkind_t memkind); +mi_memid_t _mi_memid_none(void); +mi_memid_t _mi_memid_create_os(void* base, size_t size, bool committed, bool is_zero, bool is_large); +mi_memid_t _mi_memid_create_static(void* p, size_t size); +mi_memid_t _mi_memid_create_malloc(void* p, size_t size, bool iszero); +size_t _mi_memid_size(mi_memid_t memid); + // ------------------------------------------------------ // Mimalloc pages contain allocated blocks @@ -411,7 +426,7 @@ typedef size_t mi_page_flags_t; // Points to a list of blocks that are freed by other threads. // The least-bit is set if the page is owned by the current thread. (`mi_page_is_owned`). // Ownership is required before we can read any non-atomic fields in the page. -// This way we can push a block on the thread free list and try to claim ownership atomically in `free.c:mi_free_block_mt`. +// This way we can push a block on the thread free list and try to claim ownership atomically in `alloc/free.c:mi_free_block_mt`. typedef uintptr_t mi_thread_free_t; // We store the currently used block count together with the total malloc call count as 16-bit numbers. @@ -420,7 +435,7 @@ typedef uintptr_t mi_thread_free_t; // is a separate field in `mi_page_t` but on 64-bit we use the upper 32-bits to store it. // We need the `alloc_count` and `last_used` to efficiently calculate allocation and free statistics even // in a release build; this way we can update the stats in the slow path (`_mi_page_update_stats`). -typedef union mi_used_s { +typedef union mi_used_s { size_t used_alloc; // used + alloc_count // the following struct is unused but nice for debugging struct { @@ -428,9 +443,9 @@ typedef union mi_used_s { uint16_t alloc_count; #if MI_SIZE_SIZE >= 8 uint16_t last_used; - uint16_t last_alloc; + uint16_t last_alloc; #endif - } debug_le; + } le; // little-endian layout } mi_used_t; static inline size_t mi_xused_used_count(mi_used_t xused) { return (xused.used_alloc & 0xFFFF); } @@ -462,29 +477,29 @@ static inline mi_used_t mi_xused_used_reset(mi_used_t xused) { xused.used_alloc // that case the `xthreadid` is 0 or 4 (4 is for abandoned pages that // are in the `pages_abandoned` lists of an arena, these are called "mapped" abandoned pages). // - page flags are in the bottom 3 bits of `xthread_id` for the fast path in `mi_free`. -// - The layout below is optimized for `free.c:mi_free` and `alloc.c:mi_page_alloc` +// - The layout below is optimized for `alloc/free.c:mi_free` and `alloc/alloc.c:mi_page_alloc` -- do not change it without careful consideration. -typedef struct mi_page_s { +typedef struct mi_page_s { #if (MI_PAGE_META_IS_ALIGNED) _Atomic(struct mi_page_s*) self; // points to the actual page info (for pages that span multiple slices) #endif _Atomic(mi_threadid_t) xthread_id; // thread this page belongs to. (= `theap->thread_id (or 0 or 4 if abandoned) | page_flags`) mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) + mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) + size_t block_size; // const: size available in each block (always `>0`) mi_used_t xused; // number of blocks in use (including blocks in `thread_free`) (and the allocated count for statistics) #if MI_SIZE_SIZE < 8 uint16_t xlast_used; // for statistics; on 64-bit platforms it is in bits 32..47 of xused. uint16_t xlast_alloc; // for sampling; on 64-bit platforms it is in bits 48..63 of xused. #endif - mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) - - size_t block_size; // const: size available in each block (always `>0`) + size_t page_offset; // const: relative offset from the page to the start of the blocks uint16_t capacity; // number of blocks committed uint16_t reserved; // number of blocks reserved in memory uint16_t slice_pcommitted; // committed size in OS page sizes relative to the first arena slice of the page data (or 0 if the page is fully committed already) uint8_t retire_expire; // expiration count for retired blocks bool free_is_zero; // `true` if the blocks in the free list are zero initialized - + // next cache line _Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads (= `mi_block_t* | (1 if owned)`) mi_theap_t* theap; // the theap owning this page (may not be valid or NULL for abandoned pages) @@ -493,10 +508,16 @@ typedef struct mi_page_s { struct mi_page_s* next; // next page owned by the theap with the same `block_size` struct mi_page_s* prev; // previous page owned by the theap with the same `block_size` mi_memid_t memid; // const: provenance of the page memory - + #if (MI_ENCODE_FREELIST || MI_PADDING) uintptr_t keys[MI_PAGE_KEY_COUNT]; // const: one or two random keys to encode the free lists (see `_mi_block_next`) or padding canary #endif + #if MI_QSBR + struct mi_page_s* qsbr_next; + struct mi_page_s* qsbr_prev; + bool qsbr_in_queue; + mi_page_qsbr_t qsbr; + #endif } mi_page_t; // ------------------------------------------------------ @@ -509,22 +530,33 @@ typedef struct mi_page_s { #define MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4*MI_KiB) // also aligns any multiple of this size to avoid TLB misses. #define MI_PAGE_MAX_OVERALLOC_ALIGN MI_ARENA_SLICE_SIZE // (64 KiB) limit for which we overallocate in arena pages, beyond this use OS allocation +// Offset the first block in pages with larger blocks (using the unused space at the end of the page) spread across cache sets (issue #1121). +#if !defined(MI_PAGE_CACHE_OFFSET) +#define MI_PAGE_CACHE_OFFSET 1 +#endif +#define MI_PAGE_CACHE_OFFSET_UNIT (128) // offset granularity (power of 2 between 16 and 4KiB) +#define MI_PAGE_CACHE_OFFSET_MAX (MI_PAGE_OSPAGE_BLOCK_ALIGN2 - MI_PAGE_CACHE_OFFSET_UNIT) // maximal offset (32 possible offsets) +// Offset the first block in pages with small blocks (< 512) +#define MI_PAGE_SMALL_CACHE_OFFSET_UNIT (64) +#define MI_PAGE_SMALL_CACHE_OFFSET_MAX (7*MI_PAGE_SMALL_CACHE_OFFSET_UNIT) + // The max object sizes are intended to not waste more than ~ 12.5% internally over the page sizes. #define MI_SMALL_MAX_OBJ_SIZE ((MI_SMALL_PAGE_SIZE-MI_PAGE_OSPAGE_BLOCK_ALIGN2)/6) // = 10 KiB #if MI_ENABLE_LARGE_PAGES -#define MI_MEDIUM_MAX_OBJ_SIZE ((MI_MEDIUM_PAGE_SIZE-MI_PAGE_OSPAGE_BLOCK_ALIGN2)/6) // ~ 84 KiB +#define MI_MEDIUM_MAX_OBJ_SIZE ((MI_MEDIUM_PAGE_SIZE-MI_PAGE_OSPAGE_BLOCK_ALIGN2)/6) // ~ 84 KiB (but see `alloc/page.h:mi_block_size_is_medium`) #define MI_LARGE_MAX_OBJ_SIZE (MI_LARGE_PAGE_SIZE/8) // <= 512 KiB. note: this must be a nice power of 2 or we get rounding issues with `_mi_bin` #else #define MI_MEDIUM_MAX_OBJ_SIZE (MI_MEDIUM_PAGE_SIZE/8) // <= 64 KiB #define MI_LARGE_MAX_OBJ_SIZE MI_MEDIUM_MAX_OBJ_SIZE // note: this must be a nice power of 2 or we get rounding issues with `_mi_bin` #endif #define MI_LARGE_MAX_OBJ_WSIZE (MI_LARGE_MAX_OBJ_SIZE/MI_SIZE_SIZE) +#define MI_MAX_RECLAIM_OBJ_SIZE (MI_LARGE_MAX_OBJ_SIZE/4) // only reclaim abandoned pages on free up to this block size #if (MI_LARGE_MAX_OBJ_WSIZE >= 655360) #error "mimalloc internal: define more bins" #endif -// static invariant: MI_MAX_SINGLETON_BIN >= _mi_bin(MI_LARGE_MAX_OBJ_SIZE) (See init.c for the size bins) +// static invariant: MI_MAX_SINGLETON_BIN >= _mi_bin(MI_LARGE_MAX_OBJ_SIZE) (See heap/init.c for the size bins) #if (MI_LARGE_MAX_OBJ_WSIZE <= 8192) // 64 KiB #define MI_MAX_SINGLETON_BIN (48) #elif (MI_LARGE_MAX_OBJ_WSIZE <= 32768) // 256KiB @@ -580,6 +612,7 @@ typedef struct mi_page_queue_s { mi_page_t* last; size_t count; size_t block_size; + size_t xcollect_score; // how often full pages here revived by cross-thread frees } mi_page_queue_t; // Random context @@ -617,15 +650,16 @@ struct mi_theap_s { _Atomic(mi_heap_t*) heap; // the heap this theap belongs to. _Atomic(mi_subproc_t*)subproc; // subproc this belongs too (always `subproc == heap->subproc` but needed for safe destruction) _Atomic(size_t) refcount; // reference count (needed for safe heap destroy) - + // config long page_full_retain; // how many full pages can be retained per queue (before abandoning them) bool allow_page_reclaim; // `true` if this theap can reclaim abandoned pages bool allow_page_abandon; // `true` if this theap can abandon pages to reduce memory footprint bool is_detached; // `true` if `tld->thread_id == MI_THREADID_DETACHED` + bool profile_never; // if `true`, this theap is never profiled (only the empty and meta theaps) // sampling - size_t sample_countdown; // sample countdown in requested bytes (don't change the field order; see `internal.h:_mi_theap_get_free_small_page`) + size_t sample_countdown; // sample countdown in requested bytes (don't change the field order; see `internal.h:mi_theap_get_free_small_page`) size_t sample_rate; // current sampling rate in requested bytes (or 0 to disable) (for profiling and guarded mode) uint64_t sample_requested; // total allocated/requested bytes since the last sample size_t profile_sample_rate; // sampling rate in requested bytes for profiling @@ -634,11 +668,14 @@ struct mi_theap_s { size_t guarded_sample_countdown; // countdown in requested bytes for guarded objects size_t guarded_size_min; // minimal size for guarded objects size_t guarded_size_max; // maximal size for guarded objects - + uintptr_t profiler_reserved1; // two fields available for a profiler (to avoid thread locals) + uintptr_t profiler_reserved2; + // stats unsigned long long heartbeat; // monotonic heartbeat count mi_random_ctx_t random; // random number context used for secure allocation size_t page_count; // total number of pages in the `pages` queues. + size_t page_count_os; // number of non-arena pages in the `pages` queues size_t page_retired_min; // smallest retired index (retired pages are fully free, but still in the page queues) size_t page_retired_max; // largest retired index into the `pages` array. size_t pages_full_size; // optimization: total size of blocks in the pages of the full queue (issue #1220) @@ -655,6 +692,10 @@ struct mi_theap_s { mi_page_queue_t pages[MI_BIN_COUNT]; // queue of pages for each size class (or "bin") mi_memid_t memid; // provenance of the theap struct itself (meta or os) mi_stats_t stats; // thread-local statistics + #if MI_QSBR + mi_page_t* qsbr_first; + mi_page_t* qsbr_last; + #endif }; @@ -669,7 +710,7 @@ struct mi_theap_s { struct mi_arena_pages_s; typedef struct mi_arena_pages_s mi_arena_pages_t; -#define MI_MAX_ARENAS (160) // Limited for now (and takes up .bss).. but arena's scale up exponentially (see `mi_arena_reserve`) +#define MI_MAX_ARENAS (160) // Limited for now (and takes up .bss).. but arena's scale up exponentially (see `_mi_arena_reserve`) // 160 arenas is enough for ~2 TiB memory // A dynamic thread-local variable; 0 for an invalid thread-local @@ -683,10 +724,15 @@ typedef struct mi_heap_s { mi_thread_local_t theap; // dynamic thread local for the thread-local theaps of this heap mi_arena_t* exclusive_arena; // if the heap should only allocate from a specific arena (or NULL) + long reclaim_on_free; // reclaim policy captured at heap initialization int numa_node; // if >=0, prefer this numa node for allocations + #if MI_QSBR + bool qsbr_enabled; // immutable after heap initialization + #endif + _Atomic(size_t) debug_fill_preserve; // in debug mode, the first N bytes of a block are not filled with MI_DEBUG_UNINIT/FREED mi_theap_t* theaps; // list of all thread-local theaps belonging to this heap (using the `hnext`/`hprev` fields) - mi_lock_t theaps_lock; // lock for the theaps list operations + mi_lock_t theaps_lock; // protects the theaps list and serializes heap walks with thread-exit abandonment _Atomic(mi_profiler_t*) profiler; @@ -698,6 +744,9 @@ typedef struct mi_heap_s { mi_lock_t arena_pages_lock; // lock to update the arena_pages array mi_memid_t memid; // provenance of the heap memory mi_stats_t stats; // statistics for this heap; periodically updated by merging from each theap + #if MI_QSBR + mi_heap_qsbr_t qsbr; + #endif } mi_heap_t; @@ -722,7 +771,7 @@ struct mi_subproc_s { mi_decl_align(8) // needed on some 32-bit platforms _Atomic(int64_t) purge_expire; // expiration is set if any arenas can be purged - _Atomic(mi_heap_t*) heap_main; // main heap for this sub process + _Atomic(mi_heap_t*) heap_main; // main heap for this sub process mi_heap_t* heaps; // heaps belonging to this sub-process mi_lock_t heaps_lock; @@ -760,6 +809,7 @@ struct mi_tld_s { mi_theap_t* theaps; // list of theaps in this thread (so we can abandon all when the thread terminates) mi_lock_t theaps_lock; // lock as the theaps list is sometimes accessed from another thread (on `mi_heap_free`) bool recurse; // true if deferred was called; used to prevent infinite recursion. + bool in_profiler; // true while in a profiler `on_alloc` callback; nested allocations are not sampled bool is_in_threadpool; // true if this thread is part of a threadpool (and can run arbitrary tasks) mi_memid_t memid; // provenance of the tld memory itself (meta or OS) }; @@ -781,8 +831,8 @@ struct mi_tld_s { #define MI_ARENA_MIN_SIZE (MI_BCHUNK_BITS * MI_ARENA_SLICE_SIZE) // 32 MiB (or 8 MiB on 32-bit) #define MI_ARENA_MAX_SIZE (MI_BITMAP_MAX_BIT_COUNT * MI_ARENA_SLICE_SIZE) // 16 GiB -typedef struct mi_bitmap_s mi_bitmap_t; // atomic bitmap (defined in `src/bitmap.h`) -typedef struct mi_bbitmap_s mi_bbitmap_t; // atomic binned bitmap (defined in `src/bitmap.h`) +typedef struct mi_bitmap_s mi_bitmap_t; // atomic bitmap (defined in `src/arena/bitmap.h`) +typedef struct mi_bbitmap_s mi_bbitmap_t; // atomic binned bitmap (defined in `src/arena/bitmap.h`) struct mi_arena_pages_s { mi_bitmap_t* pages; // all registered pages (abandoned and owned) diff --git a/system/lib/mimalloc/include/mimalloc/util/libc.h b/system/lib/mimalloc/include/mimalloc/util/libc.h new file mode 100644 index 0000000000000..d5709db7c20da --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/util/libc.h @@ -0,0 +1,226 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_INTERNAL_LIBC_H +#define MI_INTERNAL_LIBC_H + +#include "../types.h" + +#include // memset, memcpy, memcmp +#include // va_list, va_start, va_end, va_copy + +int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args); +int _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...); +char _mi_toupper(char c); +int _mi_strnicmp(const char* s, const char* t, size_t n); +bool _mi_strlcpy(char* dest, const char* src, size_t dest_size); // returns true if the entire src was copied +bool _mi_strlcat(char* dest, const char* src, size_t dest_size); // returns true if the entire src was appended +size_t _mi_strlen(const char* s); +size_t _mi_strnlen(const char* s, size_t max_len); +char* _mi_strnstr(char* s, size_t max_len, const char* pat); +const char* _mi_strchr(const char* s, char c); +const char* _mi_strrchr(const char* s, char c); +bool _mi_streq(const char* s, const char* t); +bool _mi_strntol(const char* s, size_t max_len, const char** end, long* result); // parses at most max_len bytes; no terminator required +int _mi_getenv(const char* name, char* result, size_t result_size); +void _mi_detect_cpu_features(void); + + +// minimum +static inline size_t mi_min(size_t x, size_t y) { return (x <= y ? x : y); } + +// maximum +static inline size_t mi_max(size_t x, size_t y) { return (x >= y ? x : y); } + +// Is `x` a power of two? (0 is considered a power of two) +static inline bool mi_is_power_of_two(uintptr_t x) { + return ((x & (x - 1)) == 0); +} + +// valid alignment values are as posix memalign: +static inline bool mi_alignment_is_valid(size_t alignment) { + return ((alignment!=0) && mi_is_power_of_two(alignment)); +} + +// Is a pointer aligned? +static inline bool mi_is_aligned(const void* p, size_t alignment) { + return (alignment==0 || ((uintptr_t)p % alignment) == 0); +} + +// Align upwards +static inline uintptr_t mi_align_up(uintptr_t sz, size_t alignment) { + mi_assert_internal(alignment != 0); + const uintptr_t mask = alignment - 1; + if ((alignment & mask) == 0) { // power of two? + return ((sz + mask) & ~mask); + } + else { + return (((sz + mask)/alignment)*alignment); + } +} + +// Align a pointer upwards +static inline void* mi_align_up_ptr(const void* p, size_t alignment) { + return (void*)mi_align_up((uintptr_t)p, alignment); +} + +// Align down +static inline uintptr_t mi_align_down(uintptr_t sz, size_t alignment) { + mi_assert_internal(alignment != 0); + const uintptr_t mask = alignment - 1; + if ((alignment & mask) == 0) { // power of two? + return (sz & ~mask); + } + else { + return ((sz/alignment)*alignment); + } +} + +// Align a pointer downwards +static inline void* mi_align_down_ptr(const void* p, size_t alignment) { + return (void*)mi_align_down((uintptr_t)p, alignment); +} + +// Divide upwards: `s <= mi_divide_up(s,d)*d < s+d`. +static inline uintptr_t mi_divide_up(uintptr_t size, size_t divider) { + mi_assert_internal(divider != 0); + return (divider == 0 ? size : ((size + divider - 1) / divider)); +} + + +// clamp an integer +static inline size_t mi_clamp(size_t sz, size_t min, size_t max) { + if (sz < min) return min; + else if (sz > max) return max; + else return sz; +} + +// Overflow detecting multiply +#if __has_builtin(__builtin_umul_overflow) || (defined(__GNUC__) && (__GNUC__ >= 5)) +#include // UINT_MAX, ULONG_MAX +#if defined(_CLOCK_T) // for Illumos +#undef _CLOCK_T +#endif +static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { + #if (SIZE_MAX == UINT_MAX) + return __builtin_umul_overflow(count, size, (unsigned int *)total); + #elif (SIZE_MAX == ULONG_MAX) + return __builtin_umull_overflow(count, size, (unsigned long *)total); + #else + return __builtin_umulll_overflow(count, size, (unsigned long long *)total); + #endif +} +#else /* __builtin_umul_overflow is unavailable */ +static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { + *total = count*size; + if mi_likely(((size|count)>>(4*MI_SIZE_SIZE))==0) { // did size and count fit both in the lower half bits of a size_t? + return false; + } + else { + return (size!=0 && (SIZE_MAX / size) < count); + } +} +#endif + +// initialize a local variable to zero; use memset as compilers optimize constant sized memset's +#define _mi_memzero_var(x) memset(&x,0,sizeof(x)) + +// Is memory zero initialized? +static inline bool mi_mem_is_zero(const void* p, size_t size) { + for (size_t i = 0; i < size; i++) { + if (((uint8_t*)p)[i] != 0) return false; + } + return true; +} + +// --------------------------------------------------------------------------------- +// Provide our own `mi_memcpy/set` for potential performance optimizations. +// --------------------------------------------------------------------------------- + +static inline int mi_memcmp(const void* dst, const void* src, size_t n) { + return memcmp(dst, src, n); +} + +static inline void* mi_memcpy(void* dst, const void* src, size_t n) { + return memcpy(dst, src, n); +} + +static inline void* mi_memset(void* dst, int val, size_t n) { + return memset(dst, val, n); +} + +static inline void* mi_memset_backward(void* dst, int val, size_t n) { + memset((uint8_t*)dst - n, val, n); + return dst; +} + +static inline void* mi_memzero(void* dst, size_t n) { + return mi_memset(dst, 0, n); +} + +static inline void* mi_memzero_backward(void* dst, size_t n) { + return mi_memset_backward(dst, 0, n); +} + +static inline void* mi_memcpy_aligned(void* dst, const void* src, size_t n) { + // on gcc/clang we can provide a hint that the pointers are word aligned. + mi_assert_internal(mi_is_aligned(dst,MI_SIZE_SIZE) && mi_is_aligned(src,MI_SIZE_SIZE)); + void* adst = mi_assume_aligned(dst, MI_SIZE_SIZE); + const void* asrc = mi_assume_aligned(src, MI_SIZE_SIZE); + return mi_memcpy(adst, asrc, n); +} + +static inline void* mi_memset_aligned(void* dst, int val, size_t n) { + mi_assert_internal(mi_is_aligned(dst,MI_SIZE_SIZE)); + void* adst = mi_assume_aligned(dst, MI_SIZE_SIZE); + return mi_memset(adst, val, n); +} + +static inline void* mi_memzero_aligned(void* dst, size_t n) { + return mi_memset_aligned(dst, 0, n); +} + + +// Zero a block: blocks are always aligned with a positive bsize in machine-word bytes. +static mi_decl_forceinline void* mi_memzero_block(mi_block_t* dst, size_t bsize) { + mi_assert_internal(bsize%MI_SIZE_SIZE == 0); + mi_assert_internal(bsize > 0); + mi_assert_internal(mi_is_aligned(dst,MI_SIZE_SIZE)); + mi_assert_internal(bsize < MI_MAX_ALIGN_SIZE || mi_is_aligned(dst,MI_MAX_ALIGN_SIZE)); + + // fast memzero for small sizes based on overlapping writes (and assuming non-zero size_t-multiple size, and size_t aligned) + // assumes constant memset(p,0,N) gets optimized to fast simd stores by the compiler + // note: disabled on riscv for now as a constant memset is not always replaced correctly by current compilers. + // (compile with -DMI_USE_MEMZERO16X=0 to disable this) + #if (!defined(MI_USE_MEMZERO16X) && !MI_ARCH_RISCV) || (MI_USE_MEMZERO16X != 0) // 16x MI_SIZE_SIZE (128 bytes on 64-bit) + if mi_unlikely(bsize < 2*MI_SIZE_SIZE) { // bsize < 16 (8) + *((size_t*)dst) = 0; + return dst; + } + mi_assert_internal(mi_is_aligned(dst,MI_MAX_ALIGN_SIZE)); + uint8_t* const start = (uint8_t*)mi_assume_aligned(dst, MI_MAX_ALIGN_SIZE); + uint8_t* const end = start + bsize; // note: if bsize is always a multiple of 16 then end is always aligned as well (but due to padding this does not hold) + if mi_likely(bsize < 8*MI_SIZE_SIZE) { // bsize < 64 (32) + const size_t ofs = (bsize>>1)&(2*MI_SIZE_SIZE); mi_assert_internal(bsize < 4*MI_SIZE_SIZE ? ofs==0 : ofs==2*MI_SIZE_SIZE); // ofs == 16 (8) + mi_memzero(start, 2*MI_SIZE_SIZE); + mi_memzero(start+ofs, 2*MI_SIZE_SIZE); + mi_memzero_backward(end-ofs, 2*MI_SIZE_SIZE); + mi_memzero_backward(end, 2*MI_SIZE_SIZE); + return dst; + } + if mi_likely(bsize <= 16*MI_SIZE_SIZE) { // bsize < 128 (64) + mi_memzero(start, 8*MI_SIZE_SIZE); + mi_memzero_backward(end, 8*MI_SIZE_SIZE); + return dst; + } + #endif + // fallback to regular memset for larger sizes + void* const wdst = mi_assume_aligned(dst,MI_SIZE_SIZE); + return mi_memzero_aligned(wdst, bsize); +} + +#endif diff --git a/system/lib/mimalloc/include/mimalloc/util/options.h b/system/lib/mimalloc/include/mimalloc/util/options.h new file mode 100644 index 0000000000000..4334dcd8df040 --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/util/options.h @@ -0,0 +1,43 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_INTERNAL_OPTIONS_H +#define MI_INTERNAL_OPTIONS_H + +#include "../types.h" + +void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message); +void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...); +void _mi_raw_message(const char* fmt, ...); +void _mi_warning_message(const char* fmt, ...); +void _mi_verbose_message(const char* fmt, ...); +void _mi_trace_message(const char* fmt, ...); +void _mi_options_init(void); +void _mi_options_post_init(void); +long _mi_option_get_fast(mi_option_t option); +void _mi_error_message(int err, const char* fmt, ...); +mi_block_t* _mi_block_next_is_corrupted(const mi_page_t* page, const mi_block_t* block, const mi_block_t* next); + +/* ----------------------------------------------------------- + Options (exposed for the debugger) +----------------------------------------------------------- */ +typedef enum mi_option_init_e { + MI_OPTION_UNINIT, // not yet initialized + MI_OPTION_DEFAULTED, // not found in the environment, use default value + MI_OPTION_INITIALIZED // found in environment or set explicitly +} mi_option_init_t; + +typedef struct mi_option_desc_s { + long value; // the value + mi_option_init_t init; // is it initialized yet? (from the environment) + mi_option_t option; // for debugging: the option index should match the option + const char* name; // option name without `mimalloc_` prefix + const char* legacy_name; // potential legacy option name +} mi_option_desc_t; + + +#endif diff --git a/system/lib/mimalloc/include/mimalloc/util/random.h b/system/lib/mimalloc/include/mimalloc/util/random.h new file mode 100644 index 0000000000000..458835accc14d --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/util/random.h @@ -0,0 +1,44 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_INTERNAL_RANDOM_H +#define MI_INTERNAL_RANDOM_H + +#include "../types.h" + +void _mi_random_init(mi_random_ctx_t* ctx); +void _mi_random_init_weak(mi_random_ctx_t* ctx); +void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx); +void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx); +size_t _mi_random_next(mi_random_ctx_t* ctx); +size_t _mi_os_random_weak(size_t extra_seed); + +// ------------------------------------------------------------------- +// Fast "random" shuffle +// ------------------------------------------------------------------- + +static inline size_t mi_random_shuffle(size_t x) { + if (x==0) { x = 17; } // ensure we don't get stuck in generating zeros +#if (MI_SIZE_SIZE>=8) + // by Sebastiano Vigna, see: + x ^= x >> 30; + x *= 0xbf58476d1ce4e5b9UL; + x ^= x >> 27; + x *= 0x94d049bb133111ebUL; + x ^= x >> 31; +#elif (MI_SIZE_SIZE==4) + // by Chris Wellons, see: + x ^= x >> 16; + x *= 0x7feb352dUL; + x ^= x >> 15; + x *= 0x846ca68bUL; + x ^= x >> 16; +#endif + return x; +} + +#endif diff --git a/system/lib/mimalloc/include/mimalloc/util/stats.h b/system/lib/mimalloc/include/mimalloc/util/stats.h new file mode 100644 index 0000000000000..b46979d9c7445 --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/util/stats.h @@ -0,0 +1,76 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_INTERNAL_STATS_H +#define MI_INTERNAL_STATS_H + +#include "../types.h" + +void _mi_stats_init(void); +void _mi_stats_merge_into(mi_stats_t* to, mi_stats_t* from); + +mi_msecs_t _mi_clock_now(void); +mi_msecs_t _mi_clock_end(mi_msecs_t start); +mi_msecs_t _mi_clock_start(void); + +/* ----------------------------------------------------------- + Statistics (in `util/stats.c`) +----------------------------------------------------------- */ + +// add to stat keeping track of the peak +void __mi_stat_increase(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_decrease(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_increase_mt(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_decrease_mt(mi_stat_count_t* stat, uint64_t amount); + +// adjust stat in special cases to compensate for double counting (and does not adjust peak values and can decrease the total) +void __mi_stat_adjust_increase(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_adjust_decrease(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, uint64_t amount); + +// counters can just be increased +static inline void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); + mi_atomic_volatile_addi64_relaxed(&stat->total, (int64_t)amount); +} + +static inline void __mi_stat_counter_increase(mi_stat_counter_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); + stat->total += (int64_t)amount; +} + +static inline void __mi_stat_counter_decrease(mi_stat_counter_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); + stat->total -= (int64_t)amount; +} + +#define mi_heap_stat_counter_increase(heap,stat,amount) __mi_stat_counter_increase_mt( &(heap)->stats.stat, amount) +#define mi_heap_stat_increase(heap,stat,amount) __mi_stat_increase_mt( &(heap)->stats.stat, amount) +#define mi_heap_stat_decrease(heap,stat,amount) __mi_stat_decrease_mt( &(heap)->stats.stat, amount) +#define mi_heap_stat_adjust_increase(heap,stat,amnt) __mi_stat_adjust_increase_mt( &(heap)->stats.stat, amnt) +#define mi_heap_stat_adjust_decrease(heap,stat,amnt) __mi_stat_adjust_decrease_mt( &(heap)->stats.stat, amnt) + +#define mi_subproc_stat_counter_increase(subproc,stat,amount) __mi_stat_counter_increase_mt( &(subproc)->stats.stat, amount) +#define mi_subproc_stat_increase(subproc,stat,amount) __mi_stat_increase_mt( &(subproc)->stats.stat, amount) +#define mi_subproc_stat_decrease(subproc,stat,amount) __mi_stat_decrease_mt( &(subproc)->stats.stat, amount) +#define mi_subproc_stat_adjust_increase(subproc,stat,amount) __mi_stat_adjust_increase_mt( &(subproc)->stats.stat, amount) +#define mi_subproc_stat_adjust_decrease(subproc,stat,amount) __mi_stat_adjust_decrease_mt( &(subproc)->stats.stat, amount) + +#define mi_theap_stat_counter_increase(theap,stat,amount) __mi_stat_counter_increase( &(theap)->stats.stat, amount) +#define mi_theap_stat_counter_decrease(theap,stat,amount) __mi_stat_counter_decrease( &(theap)->stats.stat, amount) +#define mi_theap_stat_increase(theap,stat,amount) __mi_stat_increase( &(theap)->stats.stat, amount) +#define mi_theap_stat_decrease(theap,stat,amount) __mi_stat_decrease( &(theap)->stats.stat, amount) +#define mi_theap_stat_adjust_increase(theap,stat,amnt) __mi_stat_adjust_increase( &(theap)->stats.stat, amnt) +#define mi_theap_stat_adjust_decrease(theap,stat,amnt) __mi_stat_adjust_decrease( &(theap)->stats.stat, amnt) + +#define mi_theapx_stat_counter_increase(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_counter_increase(theap,stat,amount); } else { mi_heap_stat_counter_increase(heap,stat,amount); } +#define mi_theapx_stat_adjust_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_adjust_decrease(theap,stat,amount); } else { mi_heap_stat_adjust_decrease(heap,stat,amount); } +#define mi_theapx_stat_increase(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_increase(theap,stat,amount); } else { mi_heap_stat_increase(heap,stat,amount); } +#define mi_theapx_stat_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_decrease(theap,stat,amount); } else { mi_heap_stat_decrease(heap,stat,amount); } + +#endif diff --git a/system/lib/mimalloc/include/mimalloc/util/threadlocal.h b/system/lib/mimalloc/include/mimalloc/util/threadlocal.h new file mode 100644 index 0000000000000..f0207cdb5e6e5 --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/util/threadlocal.h @@ -0,0 +1,23 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_INTERNAL_THREADLOCAL_H +#define MI_INTERNAL_THREADLOCAL_H + +#include "../types.h" + +#define mi_thread_local_key_fast ((mi_thread_local_t)1) + +mi_thread_local_t _mi_thread_local_create(void); +void _mi_thread_local_free( mi_thread_local_t key ); +bool _mi_thread_local_set( mi_thread_local_t key, void* val ); +void* _mi_thread_local_get( mi_thread_local_t key ); +void _mi_thread_locals_init(void); +void _mi_thread_locals_done(void); +void _mi_thread_locals_thread_done(void); + +#endif diff --git a/system/lib/mimalloc/include/readme.md b/system/lib/mimalloc/include/readme.md new file mode 100644 index 0000000000000..9947f8435e722 --- /dev/null +++ b/system/lib/mimalloc/include/readme.md @@ -0,0 +1,12 @@ +# Mimalloc Headers + +- [`mimalloc.h`](mimalloc.h): The main mimalloc API. +- [`mimalloc-override.h`](mimalloc-override.h): Macro-based redirection of standard allocation functions to mimalloc. One way to override all allocations functions is to include this file at the start of each source file (but of course, this only works +if you control all sources). +- [`mimalloc-new-delete.h`](mimalloc-new-delete.h): Global C++ `new` and `delete` overrides. Include this header in a single (!) source file in your project. +- [`mimalloc-stats.h`](mimalloc-stats.h): Statistics types and APIs for allocator statistics. +- [`mimalloc-profile.h`](mimalloc-profile.h): Allocation profiling callbacks, sampling controls, and the built-in pprof profiler API. +- [`mimalloc`](mimalloc): Internal allocator headers. + See in particular [`types.h`](mimalloc/types.h) for the type definitions of all + internal mimalloc data structures. + diff --git a/system/lib/mimalloc/readme.md b/system/lib/mimalloc/readme.md index 4c2ce601a3e87..0c423bb8acef1 100644 --- a/system/lib/mimalloc/readme.md +++ b/system/lib/mimalloc/readme.md @@ -15,8 +15,8 @@ is a general purpose allocator with excellent [performance](#performance) charac Initially developed by Daan Leijen for the runtime systems of the [Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages. -Latest release : `v3.5.3` (2026-09-16) recommended. -Latest v2 release: `v2.5.2` (2026-09-12) stable, legacy. +Latest release : `v3.6.0` (2026-10-23) recommended.\ +Latest v2 release: `v2.5.2` (2026-09-12) stable, legacy.\ Latest v1 release: `v1.15.2` (2026-09-12) legacy. mimalloc is a drop-in replacement for `malloc` and can be used in other programs @@ -24,7 +24,7 @@ without code changes, for example, on dynamically linked ELF-based systems (Linu ``` > LD_PRELOAD=/usr/lib/libmimalloc.so myprogram ``` -It also includes a way to dynamically override the default allocator in [Windows](#override_on_windows). +It also includes a way to dynamically override the default allocator in [Windows](#override_on_windows). Notable aspects of the design include: - __small and consistent__: the library is about 10k LOC using simple and @@ -60,7 +60,7 @@ Notable aspects of the design include: over our benchmarks. - __first-class heaps__: efficiently create and use multiple heaps to allocate across different regions. A heap can be destroyed at once instead of deallocating each object separately. - New: v3 has true first-class heaps where one can allocate in a heap from any thread. + New: v3 has true first-class heaps where one can allocate in a heap from any thread. - __bounded__: it does not suffer from _blowup_ \[1\], has bounded worst-case allocation times (_wcat_) (upto OS primitives), bounded space overhead (~0.2% meta-data, with low internal fragmentation), and has no internal points of contention using only atomic operations. @@ -76,20 +76,28 @@ Enjoy! ### Versions -There are three maintained versions of mimalloc. These are mostly equal except for how the OS memory is handled. -New development is mostly on v3, while v1 and v2 are maintained with security and bug fixes. +There are three maintained versions of mimalloc. These are mostly equal except for how the OS memory is handled. +New development is mostly on v3, while v1 and v2 are maintained with security and bug fixes. -- __v3__: recommended: simplifies the lock-free design of previous versions and improves sharing of - memory between threads. On certain large workloads this version may use - (much) less memory. Also supports true first-class heaps (that can allocate from any thread) +- __v3__: recommended: simplifies the lock-free design of previous versions and improves sharing of + memory between threads. On certain large workloads this version may use + (much) less memory. Also supports true first-class heaps (that can allocate from any thread) and has more efficient heap-walking (for the CPython GC for example). (release tags: `v3.x`, development branch `dev3`). - __v2__: stable legacy mimalloc version. Uses thread-local segments to reduce fragmentation. (release tags: `v2.x`, development branch `dev2` and `main`) -- __v1__: legacy version: initial design of mimalloc (release tags: `v1.x`, development branch `dev`). +- __v1__: legacy version: initial design of mimalloc (release tags: `v1.x`, development branch `dev`). __Send PR's against this version if possible (and otherwise `dev3`).__ ### Releases +* 2026-10-23, `v3.6.0`: (v3) add initial [profiling](#profiling) hooks and default profiler, "always on" + detailed statistics per size-bin, and always-on profiling. Initial rust bindings. Better support + for large OS page sizes, support a quiescent state based reclamation interfacec (for CPython), + better cache line aware allocation, fix heap visitor where some pages might be skipped, improve + fork support, avoid over-allocation for naturally aligned allocations, fix stderr console output + for XBox, fix huge allocation rollback, refactor the source and include files, immediately purge + very large allocations, fix bcrypt dependency on windows, fix heap-destroy bug when a fresh heap + was allocated at the same address (giving a segfault), page_reclaim_on_free is now on by default. * 2026-09-16, `v3.5.3`: (v3) critical bug fix (where the first new on a thread could fail on some platforms #1398). Fix bug with `mi_free_size` on overaligned small allocations (#1400). * 2026-09-12, `v3.5.2`, `v2.5.2`, `v1.15.2`: (v3) Reduced cache contention, improved `mi_malloc_csize`, @@ -98,37 +106,37 @@ New development is mostly on v3, while v1 and v2 are maintained with security an * 2026-09-01: Added a readme section on [getting the best performance](#getting-the-best-performance). * 2026-09-01, `v3.5.1`, `v2.5.1`, `v1.15.1`: (v3) Yet better performance for `free` calls. (v3) Added `mi_free_small(_nonnull)` for fast small allocations, and `mi_free_csize(_nonnull)` for inlined constant size free-ing. - (v3) Improved MI_OPT_ARCH options; on arm64, defaults to armv8.3 on Apple and armv8.1 otherwise. + (v3) Improved MI_OPT_ARCH options; on arm64, defaults to armv8.3 on Apple and armv8.1 otherwise. (v3) Default to `-DMI_ALLOW_THP=FULL` on Linux, where we never split transparent huge pages (THP) during purging (#1282). This may increase rss but can also improve performance. Use `-DMI_ALLOW_THP=OFF` to disable the use of THP. (v3) Improved cache behavior for small allocations. Improved riscV support. Various small build fixes. * 2026-08-18, `v1.15.0`, `v2.5.0`, `v3.5.0`: (v3) slightly better performance for `free` - using aligned chunks, cleanup cmake options, require armv8.3 (with MI_OPT_ARCH) for - faster load-acquire, increase retired page count from 1 to 3, faster double free - detection in secure mode, use faster atomics for the arena bitmaps (thank you Alan + using aligned chunks, cleanup cmake options, require armv8.3 (with MI_OPT_ARCH) for + faster load-acquire, increase retired page count from 1 to 3, faster double free + detection in secure mode, use faster atomics for the arena bitmaps (thank you Alan Andrade, #1346), faster pagemap lookup. Other: fix numa sparse node count detection (#1365), add theap stats retrieval, - use __builtin_thread_pointer on riscV (#1363), fix C mode compilation on x86 with + use __builtin_thread_pointer on riscV (#1363), fix C mode compilation on x86 with msvc (#1361), improved mingw-ucrt64 support. * 2026-08-05, `v1.9.15`, `v2.4.5`, `v3.4.5`: Fix compilation with xmalloc (#1353), make the build deterministic (#1355), - mi_zalloc_aligned fix (#763), improve support for mingw (ucrt64), fix fputs fallback on windows (#1354), (v3): use proper + mi_zalloc_aligned fix (#763), improve support for mingw (ucrt64), fix fputs fallback on windows (#1354), (v3): use proper lock backoff for first-class heap deletion, improved riscv64 codegen. -* 2026-08-01, `v1.9.14`, `v2.4.4`, `v3.4.4`: various bug and security fixes through Opus 5 LLM audit (issue #1271, by @Zoxc). +* 2026-08-01, `v1.9.14`, `v2.4.4`, `v3.4.4`: various bug and security fixes through Opus 5 LLM audit (issue #1271, by @Zoxc). (v3): use pthreads by default on macOS (issue #1333, issue #1327), fix glibc 2.44 crash (issue #1341), fix alignment check for realloc_aligned. (v1,v2,v3): Add initial mingw support, - set `errno` on allocation errors, improved double-free checks and size checks in secure mode, enable - build for universal windows platform and xbox (pr #1340), fix numa node detection when numa nodes are sparse. + set `errno` on allocation errors, improved double-free checks and size checks in secure mode, enable + build for universal windows platform and xbox (pr #1340), fix numa node detection when numa nodes are sparse. * 2026-07-20, `v3.4.3`: revert TLS slots on macOS to 108/109 (issue #1333). -* 2026-07-14, `v1.9.11`, `v2.4.1`, `v3.4.1`: various bug and security fixes through LLM audit (by @Zoxc). +* 2026-07-14, `v1.9.11`, `v2.4.1`, `v3.4.1`: various bug and security fixes through LLM audit (by @Zoxc). Fix issue with using OS memory instead of arenas for > 4GiB memory usage (v3), fix concurrency bug in concurrent heap destroy (v3), detect riscV virtual address bits at runtime, add riscV TLS support, reduce spinlock waits (v2), - use TLS slots 126/127 on macOS (v3), and other small fixes. All metadata is now separated from heap objects in v3. -* 2026-04-29, `v1.9.10`, `v2.3.2`, `v3.3.2`: various bug and security fixes through LLM audit (by @Zoxc). + use TLS slots 126/127 on macOS (v3), and other small fixes. All metadata is now separated from heap objects in v3. +* 2026-04-29, `v1.9.10`, `v2.3.2`, `v3.3.2`: various bug and security fixes through LLM audit (by @Zoxc). Only increase minimal purge size automatically if allow_thp is set to 2. Enable large OS alignment on all platforms (fixing OS large pages on Windows). Fix accounting of committed memory on Linux/macOS. Update MSVC atomics implementation when using C mode. Upstream Emscripten fixes. Proper atomic do-once - implementation. -* 2026-04-20, `v1.9.9`, `v2.3.1`, `v3.3.1`: various bug and security fixes. Special thanks to - @jinpzhanAMD, @res2k, and @GoldJohnKing for their help in improving Windows finalization, and + implementation. +* 2026-04-20, `v1.9.9`, `v2.3.1`, `v3.3.1`: various bug and security fixes. Special thanks to + @jinpzhanAMD, @res2k, and @GoldJohnKing for their help in improving Windows finalization, and @Zoxc for his help in finding various issues. * [Older release notes](#older-release-notes) @@ -218,8 +226,8 @@ The examples use the default compiler. If you like to use another, use: ## Cmake with Visual Studio -You can also use cmake on Windows. Open a Visual Studio 2022 development prompt -and invoke `cmake` with the right [generator](https://cmake.org/cmake/help/latest/generator/Visual%20Studio%2017%202022.html) +You can also use cmake on Windows. Open a Visual Studio 2022 development prompt +and invoke `cmake` with the right [generator](https://cmake.org/cmake/help/latest/generator/Visual%20Studio%2017%202022.html) and architecture, like: ``` @@ -232,7 +240,7 @@ The cmake build type is specified when actually building, for example: > cmake --build . --config=Release ``` -You can also install the [LLVM toolset](https://learn.microsoft.com/en-us/cpp/build/clang-support-msbuild?view=msvc-170#install-1) +You can also install the [LLVM toolset](https://learn.microsoft.com/en-us/cpp/build/clang-support-msbuild?view=msvc-170#install-1) on Windows to build with the `clang-cl` compiler directly: ``` @@ -365,9 +373,9 @@ Advanced options: Further options for large workloads and services: -- `MIMALLOC_ALLOW_THP=[0,1,2]`: When set to `0`, the use of transparent huge pages (THP) is disabled for the process that mimalloc +- `MIMALLOC_ALLOW_THP=[0,1,2]`: When set to `0`, the use of transparent huge pages (THP) is disabled for the process that mimalloc runs in, and otherwise it is enabled whenever THP is [enabled in the OS][linux-thp] (`[always]` or `[madvise]`). If set to 2 (default on Linux), - mimalloc also sets the `MIMALLOC_MINIMAL_PURGE_SIZE` in v3 to 2MiB to avoid potentially breaking up transparent huge pages when + mimalloc also sets the `MIMALLOC_MINIMAL_PURGE_SIZE` in v3 to 2MiB to avoid potentially breaking up transparent huge pages when purging memory which is better for performance (but may increase the rss!). On Android the default setting is 0 to reduce rss. - `MIMALLOC_USE_NUMA_NODES=N`: pretend there are at most `N` NUMA nodes. If not set, the actual NUMA nodes are detected at runtime. Setting `N` to 1 may avoid problems in some virtual environments. Also, setting it to a lower number than @@ -420,7 +428,7 @@ As always, evaluate with care as part of an overall security strategy as all of ## Debug Mode -When _mimalloc_ is built using debug mode, (`-DCMAKE_BUILD_TYPE=Debug`) (or when using `-DMI_DEBUG=ON` explicitly), +When _mimalloc_ is built using debug mode, (`-DCMAKE_BUILD_TYPE=Debug`) (or when using `-DMI_DEBUG=ON` explicitly), various checks are done at runtime to catch development errors. - Statistics are maintained in detail for each object size. They can be shown using `MIMALLOC_SHOW_STATS=1` at runtime. @@ -439,21 +447,23 @@ a guard page for every allocation may be too expensive both in terms of memory, many system calls. Therefore, there are various environment variables (and options) to tune this: - `MIMALLOC_GUARDED_SAMPLE_RATE=N`: Set the sample rate to `N` (by default 0). This mode places a guard page - behind every `N` suitable object allocations (per thread). Since the performance in guarded mode without placing - guard pages is close to release mode, this can be used to enable guard pages even in production to catch latent + behind a suitable object about every `N` requested bytes (per thread). Since the performance in guarded mode without placing + guard pages is close to release mode, this can be used to enable guard pages even in production to catch latent buffer overflow bugs. Set the sample rate to `1` to guard every object, and to `0` to place no guard pages at all. - `MIMALLOC_GUARDED_SAMPLE_SEED=N`: Start sampling at `N` (by default random). Can be used to reproduce a buffer overflow if needed. -- `MIMALLOC_GUARDED_MIN=N`, `MIMALLOC_GUARDED_MAX=N`: Minimal and maximal _rounded_ object sizes for which a guard +- `MIMALLOC_GUARDED_MIN=N`, `MIMALLOC_GUARDED_MAX=N`: Minimal and maximal _rounded_ object sizes for which a guard page is considered (`0` and `1GiB` respectively). If you suspect a buffer overflow occurs with an object of size 141, set the minimum and maximum to `148` and the sample rate to `1` to have all of those guarded. -- `MIMALLOC_GUARDED_PRECISE=1`: If we have an object of size 13, we would usually place it an aligned 16 bytes in - front of the guard page. Using `MIMALLOC_GUARDED_PRECISE` places it exactly 13 bytes before a page so that even - a 1 byte overflow is detected. This violates the C/C++ minimal alignment guarantees though so use with care. - + # Overriding Standard Malloc @@ -477,12 +487,12 @@ object file. For example: Another way to override statically that works on all platforms, is to link statically to mimalloc (as shown in the introduction) and include a header file in each source file that re-defines `malloc` etc. to `mi_malloc`. -This is provided by [`mimalloc-override.h`](include/mimalloc-override.h). This only works +This is provided by [`mimalloc-override.h`](include/mimalloc-override.h). This only works reliably though if all sources are under your control or otherwise mixing of pointers from different heaps may occur! Note: recently we also enabled static overloading on Windows. In that case you need -to link with the static CRT _release_ runtime (`/MT`) and link with the static +to link with the static CRT _release_ runtime (`/MT`) and link with the static `mimalloc(-debug).obj` (to take precendence over the definitions in the CRT library). @@ -524,9 +534,9 @@ the [shell](https://stackoverflow.com/questions/43941322/dyld-insert-libraries-i ### Dynamic Override on Windows -We use a separate redirection DLL to override mimalloc on Windows -such that we redirect all malloc/free calls that go through the (dynamic) C runtime allocator, -including those from other DLL's or libraries. As it intercepts all allocation calls on a low level, +We use a separate redirection DLL to override mimalloc on Windows +such that we redirect all malloc/free calls that go through the (dynamic) C runtime allocator, +including those from other DLL's or libraries. As it intercepts all allocation calls on a low level, it can be used on large programs that include other 3rd party components. There are four requirements to make the overriding work well: @@ -534,27 +544,27 @@ There are four requirements to make the overriding work well: 2. Link your program explicitly with the `mimalloc.dll.lib` export library for the `mimalloc.dll`. (which must be compiled with `-DMI_OVERRIDE=ON`, which is the default though). - To ensure the `mimalloc.dll` is actually loaded at run-time it is easiest + To ensure the `mimalloc.dll` is actually loaded at run-time it is easiest to insert some call to the mimalloc API in the `main` function, like `mi_version()` (or use the `/include:mi_version` switch on the linker command, or - similarly, `#pragma comment(linker, "/include:mi_version")` in some source file). - See the `mimalloc-test-override` project for an example on how to use this. + similarly, `#pragma comment(linker, "/include:mi_version")` in some source file). + See the `mimalloc-test-override` project for an example on how to use this. -3. The `mimalloc-redirect.dll` must be put in the same directory as the main +3. The `mimalloc-redirect.dll` must be put in the same directory as the main `mimalloc.dll` at runtime (as it is a dependency of that DLL). - The redirection DLL ensures that all calls to the C runtime malloc API get + The redirection DLL ensures that all calls to the C runtime malloc API get redirected to mimalloc functions (which reside in `mimalloc.dll`). 4. Ensure the `mimalloc.dll` comes as early as possible in the import list of the final executable (so it can intercept all potential allocations). You can use `minject -l ` to check this if needed. -For best performance on Windows with C++, it is also recommended to also override +For best performance on Windows with C++, it is also recommended to also override the `new`/`delete` operations (by including [`mimalloc-new-delete.h`](include/mimalloc-new-delete.h) a single(!) source file in your project). The environment variable `MIMALLOC_DISABLE_REDIRECT=1` can be used to disable dynamic -overriding at run-time. Use `MIMALLOC_VERBOSE=1` to check if mimalloc was successfully +overriding at run-time. Use `MIMALLOC_VERBOSE=1` to check if mimalloc was successfully redirected. For different platforms than x64, you may need a specific [redirection dll](bin). @@ -567,13 +577,13 @@ to patch the executable's import tables. Consider the following build options (v3): -- `-DMI_OPT_ARCH=ON` (by default ON on arm64): enables architecture specific optimizations. - This uses `-march=haswell;-march=avx2` on x64 (available since 2013), `-march=armv8.1` on arm64 (2016), - and `-march=rv64gcb_zacas` on riscV (2023). +- `-DMI_OPT_ARCH=ON` (by default ON on arm64): enables architecture specific optimizations. + This uses `-march=haswell;-march=avx2` on x64 (available since 2013), `-march=armv8.1` on arm64 (2016), + and `-march=rv64gcb_zacas` on riscV (2023). Each benefits mimalloc with better bit operations and/or native atomic operations (but also, if your - emulator or hardware does not support these extensions the program will fail with an illegal instruction). - On Apple arm64 the default is `-march=armv8.3` (which is supported on the M1 and better) which includes - efficient load-acquire instructions which mimalloc uses. For riscV the corresponding `zalasr` extension + emulator or hardware does not support these extensions the program will fail with an illegal instruction). + On Apple arm64 the default is `-march=armv8.3` (which is supported on the M1 and better) which includes + efficient load-acquire instructions which mimalloc uses. For riscV the corresponding `zalasr` extension can be enabled as `-DMI_OPT_ARCH=rv64gcb_zacas_zalasr` when your hardware supports it. - `-DMI_ALLOW_THP=OFF`: disabling THP can reduce the rss of certain programs significantly. @@ -591,7 +601,7 @@ In the source code: - Use `mi_zalloc` for zero initialized allocations. - Use the typed macros `mi_malloc_tp` etc. as these are both more safe and also use `mi_malloc_csize` internally. - + - Use `mi_malloc_csize`/`mi_free_csize` etc. when the allocated or free'd size is a constant as these are inlined and statically redirect to `mi_malloc_small`/`mi_free_small` when possible. These are also used by the C++ `mimalloc-new-delete.h` sized overrides. @@ -604,15 +614,37 @@ In the source code: For run-time systems and compilers (like Koka, Lean, Rust (?) etc.), we can do slightly better still. -- If the allocation size is statically known, use `mi_malloc_csize` and `mi_free_csize` etc. when possible. +- If the allocation size is statically known, use `mi_malloc_csize` and `mi_free_csize` etc. when possible. (or directly `mi_malloc_small`/`mi_free_small`). - If the runtime already carries thread local state, it may be faster to get the default `theap` for each thread - up-front (`mi_theap_get_default()`) and use the bestest `mi_theap_malloc(_small)` etc. passing the theap pointer directly. + up-front (`mi_theap_get_default()`) and use the bestest `mi_theap_malloc(_small)` etc. passing the theap pointer directly. This avoids having mimalloc look up the thread local pointer all the time and - skips a theap `NULL` check. Whether this is - faster depends a bit on the OS implementation of thread local variables. + skips a theap `NULL` check. Whether this is + faster depends a bit on the OS implementation of thread local variables. + + +# Profiling + +Profiling support is only recently added and may still change. +The profiling support is "always on", even in release builds and only adds overhead once profiling +is actually activated. There is a profiling API that allows the use of custom profiling tools +to hook into mimalloc at runtime, see [`mimalloc-profile.h`](include/mimalloc-profile.h). + +However, mimalloc also has a default built-in profiler that can be used with Google's +[`pprof`](https://github.com/google/pprof) tools. It can be activated using environment variables: +- `MIMALLOC_PROFILE=`: set a profile basename, this activates the profiler for the process. + This will write _pprof_ compatible `..pb` files. + (If you add the extension `.text` it writes old-style heap profiles as readable text). + You can later analyze the information information using _pprof_ tools, for example, + with a basename of `profile`, we can invoke _pprof_ in the browers as `pprof -http=:8080 ./myprogram profile.*` +- `MIMALLOC_PROFILE_ALLOC_INTERVAL=N`: set the (approximate) number of allocated bytes between saving profile snapshots + of the heap profiler. By default `N` is specified as `1GiB`. +- `MIMALLOC_PROFILE_INUSE_INTERVAL=N`: when the in-use bytes grow by this amount, take a snapshot. By default disabled (`0`). +- `MIMALLOC_PROFILE_TIME_INTERVAL=N`: number of seconds between automatic profile snapshots. By default disabled (`0`). +- `MIMALLOC_PROFILE_SAMPLE_RATE=N`: the sample rate (in bytes) for the heap profiler. Every `N` bytes (approximately, + the profiler uses a Poisson distribution to spread samples) a sample is taken at an allocation call site. By default `512KiB`. # Tools @@ -711,7 +743,7 @@ use a tool like [TraceControl] that is specialized for analyzing mimalloc traces # Performance -Last update: 2021-01-30. +Last update: 2021-01-30.\ (Note 2026-09-01: this is a while ago now, many of these allocators have improved over time as well. YMMV.) We tested _mimalloc_ against many other top allocators over a wide @@ -970,21 +1002,21 @@ provided by the bot. You will only need to do this once across all repos using o # Older Release Notes -* 2026-04-15, `v1.9.8`, `v2.3.0`, `v3.3.0`: initial support for github (binary) releases, +* 2026-04-15, `v1.9.8`, `v2.3.0`, `v3.3.0`: initial support for github (binary) releases, fix visiting of full pages during collection (performance), fix THP alignment (performance), fix arm64 cross-compilation on Windows, enable guard pages in debug mode, - always use uncommitted areas between arenas (security), enable static overloading of `malloc` etc. on Windows with the + always use uncommitted areas between arenas (security), enable static overloading of `malloc` etc. on Windows with the static CRT (by @Noxybot), fix TLS slot leak on Windows (v3), enable clean DLL load/unload with statically linked - mimalloc (v3), fix race in `mi_heap_destroy` (v3), by default put page meta info separate from allocated - objects (v3,security), fix C++ overrides for emscripten. Various bugs found by DeepTest include: + mimalloc (v3), fix race in `mi_heap_destroy` (v3), by default put page meta info separate from allocated + objects (v3,security), fix C++ overrides for emscripten. Various bugs found by DeepTest include: fix offset for `mi_heap_realloc_aligned`, fix `mi_(w)dupenv_s` buffer size, fix potential overflow in size options, - and error codes for `mi_reallocarr(ay)`. + and error codes for `mi_reallocarr(ay)`. * 2026-02-03, `v3.2.8` (rc3): Fix thread reinitialize issue on macOS. Fix SIMD codegen bug on older GCC versions. Extend Windows TLS slot limit from 64 to 1088. Report commit statistics more precise. Fixes issue in free-page search in arenas. -* 2026-01-15, `v1.9.7`, `v2.2.7`, `v3.2.7` (rc2): Fix zero initializing blocks that were OS allocated. +* 2026-01-15, `v1.9.7`, `v2.2.7`, `v3.2.7` (rc2): Fix zero initializing blocks that were OS allocated.\ For v3 various bug and performance fixes. Fix Debian 32-bit compilation. -* 2026-01-08, `v1.9.6`, `v2.2.6`, `v3.2.6` (rc1): Important bug fixes. Many improvements to v3 including +* 2026-01-08, `v1.9.6`, `v2.2.6`, `v3.2.6` (rc1): Important bug fixes. Many improvements to v3 including true first-class heaps where one can allocate in heap from any thread, and track statistics per heap as well. Added `MIMALLOC_ALLOW_THP` option. This is by default enabled except on Android. When THP is detected on v3, mimalloc will set the `MIMALLOC_MINIMAL_PURGE_SIZE` to 2MiB to avoid breaking up potential THP huge pages. @@ -992,22 +1024,22 @@ provided by the bot. You will only need to do this once across all repos using o Fixed rare race condition on older v3, fixed potential buffer overflow in debug statistics, add API for returning allocated sizes on allocation and free. * 2025-06-09, `v1.9.4`, `v2.2.4`, `v3.1.4` (beta) : Some important bug fixes, including a case where OS memory - was not always fully released. Improved v3 performance, build on XBox, fix build on Android, support interpose - for older macOS versions, use MADV_FREE_REUSABLE on macOS, always check commit success, better support for Windows + was not always fully released. Improved v3 performance, build on XBox, fix build on Android, support interpose + for older macOS versions, use MADV_FREE_REUSABLE on macOS, always check commit success, better support for Windows fixed TLS offset, etc. * 2025-03-28, `v1.9.3`, `v2.2.3`, `v3.0.3` (beta) : Various small bug and build fixes, including: - fix arm32 pre v7 builds, fix mingw build, get runtime statistics, improve statistic commit counts, - fix execution on non BMI1 x64 systems. -* 2025-03-06, `v1.9.2`, `v2.2.2`, `v3.0.2-beta`: Various small bug and build fixes. - Add `mi_options_print`, `mi_arenas_print`, and the experimental `mi_stat_get` and `mi_stat_get_json`. - Add `mi_thread_set_in_threadpool` and `mi_heap_set_numa_affinity` (v3 only). Add vcpkg portfile. + fix arm32 pre v7 builds, fix mingw build, get runtime statistics, improve statistic commit counts, + fix execution on non BMI1 x64 systems. +* 2025-03-06, `v1.9.2`, `v2.2.2`, `v3.0.2-beta`: Various small bug and build fixes. + Add `mi_options_print`, `mi_arenas_print`, and the experimental `mi_stat_get` and `mi_stat_get_json`. + Add `mi_thread_set_in_threadpool` and `mi_heap_set_numa_affinity` (v3 only). Add vcpkg portfile. Upgrade mimalloc-redirect to v1.3.2. `MI_OPT_ARCH` is off by default now but still assumes armv8.1-a on arm64 for fast atomic operations. Add QNX support. -* 2025-01-03, `v1.8.9`, `v2.1.9`, `v3.0.1-alpha`: Interim release. Support Windows arm64. New [guarded](#guarded) build that can place OS - guard pages behind objects to catch buffer overflows as they occur. +* 2025-01-03, `v1.8.9`, `v2.1.9`, `v3.0.1-alpha`: Interim release. Support Windows arm64. New [guarded](#guarded) build that can place OS + guard pages behind objects to catch buffer overflows as they occur. Many small fixes: build on Windows arm64, cygwin, riscV, and dragonfly; fix Windows static library initialization to account for - thread local destructors (in Rust/C++); macOS tag change; macOS TLS slot fix; improve stats; - consistent `mimalloc.dll` on Windows (instead of `mimalloc-override.dll`); fix mimalloc-redirect on Win11 H2; + thread local destructors (in Rust/C++); macOS tag change; macOS TLS slot fix; improve stats; + consistent `mimalloc.dll` on Windows (instead of `mimalloc-override.dll`); fix mimalloc-redirect on Win11 H2; add 0-byte to canary; upstream CPython fixes; reduce .bss size; allow fixed TLS slot on Windows for improved performance. * 2024-05-21, `v1.8.7`, `v2.1.7`: Fix build issues on less common platforms. Started upstreaming patches diff --git a/system/lib/mimalloc/src/alloc-aligned.c b/system/lib/mimalloc/src/alloc/alloc-aligned.c similarity index 93% rename from system/lib/mimalloc/src/alloc-aligned.c rename to system/lib/mimalloc/src/alloc/alloc-aligned.c index 51f29732cbd85..7895bee0851b0 100644 --- a/system/lib/mimalloc/src/alloc-aligned.c +++ b/system/lib/mimalloc/src/alloc/alloc-aligned.c @@ -6,10 +6,13 @@ terms of the MIT license. A copy of the license can be found in the file -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc/internal.h" -#include "mimalloc/prim-tls.h" // _mi_theap_default +#include "mimalloc/types.h" +#include "mimalloc/prim/prim-tls.h" // _mi_theap_default +#include "mimalloc/alloc/alloc.h" +#include "mimalloc/alloc/free.h" +#include "mimalloc/alloc/page.h" +#include "mimalloc/profile/sample-guarded.h" -#include // memset // ------------------------------------------------------ // Aligned Allocation @@ -17,13 +20,12 @@ terms of the MIT license. A copy of the license can be found in the file static bool mi_malloc_is_naturally_aligned( size_t size, size_t alignment ) { // certain blocks are always allocated at a certain natural alignment. - // (see also `arena.c:mi_arenas_page_alloc_fresh`). + // (see `alloc/page.h:mi_natural_alignment_of_size`, asserted in `arena/arena-page.c:mi_arenas_page_alloc_fresh`). mi_assert_internal(mi_alignment_is_valid(alignment)); if (alignment > size) return false; const size_t bsize = mi_good_size(size); - const bool ok = (bsize <= MI_PAGE_MAX_START_BLOCK_ALIGN2 && _mi_is_power_of_two(bsize)) || // power-of-two under N - (alignment==MI_PAGE_OSPAGE_BLOCK_ALIGN2 && (bsize % MI_PAGE_OSPAGE_BLOCK_ALIGN2)==0); // or multiple of N - if (ok) { mi_assert_internal((bsize & (alignment-1)) == 0); } // since both power of 2 and alignment <= size + const bool ok = (alignment <= mi_natural_alignment_of_size(bsize)); + if (ok) { mi_assert_internal((bsize & (alignment-1)) == 0); } // since the natural alignment always divides `bsize` return ok; } @@ -41,10 +43,10 @@ static mi_decl_noinline mi_decl_restrict void* mi_theap_malloc_guarded_aligned(m const size_t oversize = size + alignment - 1; void* const base = _mi_theap_malloc_guarded(theap, oversize, zero, ppage); if (base==NULL) return NULL; - void* const p = _mi_align_up_ptr(base, alignment); + void* const p = mi_align_up_ptr(base, alignment); mi_track_align(base, p, (uint8_t*)p - (uint8_t*)base, size); mi_assert_internal(mi_usable_size(p) >= size); - mi_assert_internal(_mi_is_aligned(p, alignment)); + mi_assert_internal(mi_is_aligned(p, alignment)); return p; } #endif @@ -81,7 +83,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap p = _mi_theap_malloc_zero(theap, oversize, zero, 0, &page); if (p == NULL) return NULL; } - mi_assert_internal(page == _mi_ptr_page(p)); + mi_assert_internal(page == mi_ptr_page(p)); if (ppage!=NULL) { *ppage = page; } // .. and align within the allocation @@ -91,7 +93,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap mi_assert_internal(adjust < alignment); void* aligned_p = (void*)((uintptr_t)p + adjust); - // note: after the above allocation, the page may be abandoned now (as it became full, see `page.c:_mi_malloc_generic`) + // note: after the above allocation, the page may be abandoned now (as it became full, see `alloc/alloc-generic.c:_mi_malloc_generic`) // and we no longer own it. We should be careful to only read constant fields in the page, // or use safe atomic access as in `mi_page_set_has_interior_pointers`. // (we can access the page though since the just allocated pointer keeps it alive) @@ -100,7 +102,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap #if MI_GUARDED || MI_PROFILE // set tag to aligned so mi_usable_size works with guard pages - // note: we could have gotten a profiled or guarded block but that is still ok + // note: we could have gotten a profiled or guarded block but that is still ok // and in such case the MI_BLOCK_TAG_ALIGNED is just not at the start of the real heap block if (adjust >= sizeof(mi_block_t)) { mi_block_t* const block = (mi_block_t*)p; @@ -117,7 +119,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap mi_assert_internal(mi_usable_size(aligned_p)>=size); mi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust); #if MI_DEBUG > 1 - mi_page_t* const apage = _mi_ptr_page(aligned_p); + mi_page_t* const apage = mi_ptr_page(aligned_p); mi_block_t* unalign_p = _mi_page_ptr_unalign(apage, aligned_p); mi_assert_internal(p == (void*)unalign_p || mi_block_ptr_is_sampled(unalign_p, aligned_p)); #endif @@ -152,11 +154,11 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t if mi_likely(!mi_theap_should_sample(theap,size)) #endif { - if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) + if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) { mi_page_t* page = NULL; void* p = _mi_theap_malloc_zero(theap, size, zero, 0,&page); - if (ppage!=NULL) { *ppage = page; } + if (ppage!=NULL) { *ppage = page; } const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0; if mi_likely(is_aligned_or_null) { return p; @@ -182,13 +184,19 @@ static mi_decl_cold mi_decl_noinline void* mi_error_bad_alignment(size_t size, s } // Primitive aligned allocation -static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept +static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept { // note: we don't require `size > offset`, we just guarantee that the address at offset is aligned regardless of the allocated size. if mi_unlikely(!mi_alignment_is_valid(alignment)) { // require power-of-two and multiple of void* (see ) return mi_error_bad_alignment(size, alignment, offset); } + // Use a naturally aligned size class instead of over-allocating and adjusting the pointer. + // Keep the requested size when padding is enabled for byte-precise overflow checks. + if (MI_PADDING_SIZE == 0 && offset == 0 && size < alignment && alignment <= MI_PAGE_MAX_START_BLOCK_ALIGN2) { + size = alignment; + } + // try first if there happens to be a small block available with just the right alignment // since most small power-of-2 blocks (under MI_PAGE_MAX_BLOCK_START_ALIGN2) are already // naturally aligned this can be often the case. @@ -200,11 +208,11 @@ static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, con { #if MI_SAMPLE // check if we shouldn't take a sample if mi_likely(!mi_theap_should_sample(theap,size)) - #endif + #endif { const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` const size_t padsize = size + MI_PADDING_SIZE; - mi_page_t* page = _mi_theap_get_free_small_page(theap, padsize, false); + mi_page_t* page = mi_theap_get_free_small_page(theap, padsize, false); if mi_likely(page->free != NULL) { const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0; if mi_likely(is_aligned) @@ -346,14 +354,14 @@ static void* mi_theap_realloc_zero_aligned_at(mi_theap_t* theap, void* p, size_t void* const newp = mi_theap_malloc_zero_aligned_at(theap,newsize,alignment,offset,false/*zero?*/,&newpage); if (newp != NULL) { const size_t copy_size = (newsize > size ? size : newsize); - const size_t zero_start = (copy_size >= sizeof(intptr_t) ? copy_size - sizeof(intptr_t) : 0); // also set last word in the previous allocation to zero to ensure any padding is zero-initialized + const size_t zero_start = (copy_size >= sizeof(intptr_t) ? copy_size - sizeof(intptr_t) : 0); // also set last word in the previous allocation to zero to ensure any padding is zero-initialized const size_t usable = _mi_page_usable_size(newpage,newp); mi_assert_internal(usable >= newsize); // use usable for zero'ing, issue #763 if (zero && usable > zero_start) { // also set last word in the previous allocation to zero to ensure any padding is zero-initialized - _mi_memzero((uint8_t*)newp + zero_start, usable - zero_start); + mi_memzero((uint8_t*)newp + zero_start, usable - zero_start); } - _mi_memcpy(newp, p, copy_size); // cannot be aligned due to abitrary offset... (todo: require offset to be a multiple of sizeof(void*)?) + mi_memcpy(newp, p, copy_size); // cannot be aligned due to abitrary offset... (todo: require offset to be a multiple of sizeof(void*)?) mi_free(p); // only free if successful } return newp; @@ -443,5 +451,3 @@ mi_decl_nodiscard void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, si mi_decl_nodiscard void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { return mi_theap_recalloc_aligned(_mi_heap_theap(heap), p, newcount, size, alignment); } - - diff --git a/system/lib/mimalloc/src/alloc/alloc-generic.c b/system/lib/mimalloc/src/alloc/alloc-generic.c new file mode 100644 index 0000000000000..52e0cd8c02d05 --- /dev/null +++ b/system/lib/mimalloc/src/alloc/alloc-generic.c @@ -0,0 +1,515 @@ +/*---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +/* ----------------------------------------------------------- + The "slow" allocation path (`mi_malloc_generic`), including + finding suitable pages with free blocks. +----------------------------------------------------------- */ + +#include "mimalloc.h" +#include "mimalloc/types.h" +#include "mimalloc/prim/prim-tls.h" +#include "mimalloc/util/stats.h" +#include "mimalloc/alloc/alloc.h" +#include "mimalloc/alloc/page.h" +#include "mimalloc/profile/sample-profile.h" +#include "page-queue.h" +#include "page.h" + +// returns `true` if after collection `mi_page_immediate_available` is true. +static inline bool mi_page_free_quick_collect(mi_page_t* page) { + if mi_likely(page->free != NULL) return true; + if (page->local_free == NULL) return false; + // move local_free to free + page->free = page->local_free; + page->local_free = NULL; + page->free_is_zero = false; + _mi_page_update_sample_countdown(page); + return true; +} + + +/* ----------------------------------------------------------- + Find pages with free blocks +-------------------------------------------------------------*/ + +#define MI_XCOLLECT_SCORE_MAX (256) // saturation limit for `xcollect_score` +#define MI_XCOLLECT_SCORE_SHIFT (5) // retain bonus = score >> shift (max 8) +#define MI_XCOLLECT_SCORE_INC (8) // a revived page saves a full abandon/reclaim round-trip + + +// Find a page with free blocks of `page->block_size`. +static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, mi_page_queue_t* pq, bool first_try) +{ + // search through the pages in "next fit" order + size_t count = 0; + long candidate_limit = 0; // we reset this on the first candidate to limit the search + long page_full_retain = (pq->block_size > MI_SMALL_MAX_OBJ_SIZE ? 0 : theap->page_full_retain); // only retain small pages + if (page_full_retain >= 0) { + // adaptively retain more full pages when cross-thread frees keep reviving them (producer/consumer patterns) + page_full_retain += (long)(pq->xcollect_score >> MI_XCOLLECT_SCORE_SHIFT); + } + mi_page_t* page_candidate = NULL; // a page with free space + mi_page_t* page = pq->first; + mi_page_t* const last = pq->last; + + while (page!=NULL) + { + mi_page_t* next = page->next; // remember next (as this page can move to another queue) + count++; + candidate_limit--; + + // search up to N pages for a best candidate + + // is the local free list non-empty? + bool immediate_available = mi_page_immediate_available(page); + if (!immediate_available) { + // collect freed blocks by us and other threads to we get a proper use count + const bool collected_xfree = _mi_page_free_collect(page, false); + immediate_available = mi_page_immediate_available(page); + if (collected_xfree && immediate_available) { + // a cross-thread free revived this page: retaining such pages avoids an abandon/reclaim round-trip + pq->xcollect_score = mi_min(pq->xcollect_score + MI_XCOLLECT_SCORE_INC, MI_XCOLLECT_SCORE_MAX); + } + else if (pq->xcollect_score > 0) { + pq->xcollect_score--; + } + } + + // if the page is completely full, move it to the `mi_pages_full` + // queue so we don't visit long-lived pages too often. + if (!immediate_available && !mi_page_is_expandable(page)) { + page_full_retain--; + if (page_full_retain < 0) { + mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page)); + _mi_page_to_full(page, pq); + } + else if (page!=last && pq->last!=page) { + // avoid revisiting this page for a while + mi_page_queue_move_to_back(theap, pq, page); + } + } + else { + // the page has free space, make it a candidate + // we prefer non-expandable pages with high usage as candidates (to reduce commit, and increase chances of free-ing up pages) + if (page_candidate == NULL) { + page_candidate = page; + candidate_limit = _mi_option_get_fast(mi_option_page_max_candidates); + } + else if (mi_page_all_free(page_candidate)) { + _mi_page_free(page_candidate, pq); + page_candidate = page; + } + // prefer to reuse fuller pages (in the hope the less used page gets freed) + else if (mi_page_used(page) >= mi_page_used(page_candidate) && !mi_page_is_mostly_used(page)) { // && !mi_page_is_expandable(page)) { + page_candidate = page; + } + // if we find a non-expandable candidate, or searched for N pages, return with the best candidate + if (immediate_available || candidate_limit <= 0) { + mi_assert_internal(page_candidate!=NULL); + break; + } + } + + #if 0 + // first-fit algorithm without candidates + // If the page contains free blocks, we are done + if (mi_page_immediate_available(page) || mi_page_is_expandable(page)) { + break; // pick this one + } + + // If the page is completely full, move it to the `mi_pages_full` + // queue so we don't visit long-lived pages too often. + mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page)); + _mi_page_to_full(page, pq); + #endif + if (page==last) { page=NULL; } // don't revisit earlier pages that moved to the back + else { page = next; } + } // for each page + + mi_theap_stat_counter_increase(theap, page_searches, count); + mi_theap_stat_counter_increase(theap, page_searches_count, 1); + + // set the page to the best candidate + if (page_candidate != NULL) { + page = page_candidate; + } + if (page != NULL) { + if (!mi_page_immediate_available(page)) { + mi_assert_internal(mi_page_is_expandable(page)); + if (!_mi_page_extend_free(theap, page)) { + page = NULL; // failed to extend + } + } + mi_assert_internal(page == NULL || mi_page_immediate_available(page)); + } + + if (page == NULL) { + _mi_theap_collect_retired(theap, false); // perhaps make a page available + page = _mi_page_fresh(theap, pq); + mi_assert_internal(page == NULL || mi_page_immediate_available(page)); + if (page == NULL && first_try) { + // out-of-memory _or_ an abandoned page with free blocks was reclaimed, try once again + page = mi_page_queue_find_free_ex(theap, pq, false); + mi_assert_internal(page == NULL || mi_page_immediate_available(page)); + } + } + else { + mi_assert_internal(page == NULL || mi_page_immediate_available(page)); + // move the page to the front of the queue + mi_page_queue_move_to_front(theap, pq, page); + page->retire_expire = 0; + _mi_page_qsbr_clear(page); + // _mi_theap_collect_retired(theap, false); // update retire counts; note: increases rss on MemoryLoad bench so don't do this + } + mi_assert_internal(page == NULL || mi_page_immediate_available(page)); + + + return page; +} + +// Look for a page with free blocks of `size` but don't try to search or allocate +static inline mi_page_t* mi_page_queue_lookup_free_first(mi_theap_t* theap, mi_page_queue_t* pq) { + mi_assert_internal(!mi_page_queue_is_huge(pq)); + mi_page_t* page = pq->first; + if mi_likely(page!=NULL && mi_page_free_quick_collect(page)) { + // fast path + #if (MI_SECURE>=2) // in secure mode, we extend half the time to increase randomness + if (page->capacity < page->reserved && ((_mi_theap_random_next(theap) & 1) == 1)) { + (void)_mi_page_extend_free(theap, page); // ok if this fails + mi_assert_internal(mi_page_immediate_available(page)); + } + #else + MI_UNUSED(theap); + #endif + page->retire_expire = 0; + _mi_page_qsbr_clear(page); + mi_assert_internal(mi_page_immediate_available(page)); + mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(mi_ptr_page(mi_page_start(page))==page); + return page; + } + else { + return NULL; + } +} + +// Find a page with free blocks of `size`. +static inline mi_page_t* mi_page_queue_find_free(mi_theap_t* theap, mi_page_queue_t* pq) { + // mi_page_queue_t* pq = mi_page_queue(theap, size); + mi_assert_internal(!mi_page_queue_is_huge(pq)); + + // check the first page: we even do this with candidate search or otherwise we re-search every time + mi_page_t* page = mi_page_queue_lookup_free_first(theap,pq); + if (page==NULL) { + page = mi_page_queue_find_free_ex(theap, pq, true); + if (page==NULL) return NULL; + } + mi_assert_internal(mi_page_immediate_available(page)); + mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(mi_ptr_page(mi_page_start(page))==page); + return page; +} + +// Huge pages contain just one block, and the segment contains just that page. +// Huge pages are also use if the requested alignment is very large (> MI_BLOCK_ALIGNMENT_MAX) +// so their size is not always `> MI_LARGE_OBJ_SIZE_MAX`. +static mi_page_t* mi_huge_page_alloc(mi_theap_t* theap, size_t size, size_t page_alignment, mi_page_queue_t* pq) { + _mi_theap_qsbr_collect_delayed(theap); + const size_t block_size = _mi_os_good_alloc_size(size); + // mi_assert_internal(mi_bin(block_size) == MI_BIN_HUGE || page_alignment > 0); + #if MI_HUGE_PAGE_ABANDON + #error todo. + #else + // mi_page_queue_t* pq = mi_page_queue(theap, MI_LARGE_MAX_OBJ_SIZE+1); // always in the huge queue regardless of the block size + mi_assert_internal(mi_page_queue_is_huge(pq)); + #endif + mi_page_t* page = _mi_page_fresh_alloc(theap, pq, block_size, page_alignment); + if (page != NULL) { + mi_assert_internal(mi_page_block_size(page) >= size); + mi_assert_internal(mi_page_immediate_available(page)); + mi_assert_internal(mi_page_is_huge(page)); + mi_assert_internal(mi_page_is_singleton(page)); + #if MI_HUGE_PAGE_ABANDON + mi_assert_internal(mi_page_is_abandoned(page)); + mi_page_set_theap(page, NULL); + #endif + // mi_theap_stat_increase(theap, malloc_huge, mi_page_block_size(page)); + // mi_theap_stat_counter_increase(theap, malloc_huge_count, 1); + } + return page; +} + +// Allocate a page +// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed. +static mi_page_t* mi_find_page(mi_theap_t* theap, size_t size, size_t huge_alignment) mi_attr_noexcept { + const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` + if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE) { // TODO: remove as we check in generic_fallback ? + _mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size); + return NULL; + } + mi_page_queue_t* pq = mi_page_queue(theap, (huge_alignment > 0 ? MI_LARGE_MAX_OBJ_SIZE+1 : size)); + mi_page_t* page; + // huge allocation? + if mi_unlikely(mi_page_queue_is_huge(pq) || req_size > MI_MAX_ALLOC_SIZE) { + page = mi_huge_page_alloc(theap,size,huge_alignment,pq); + } + else { + // otherwise find a page with free blocks in our size segregated queues + #if MI_PADDING + mi_assert_internal(size >= MI_PADDING_SIZE); + #endif + page = mi_page_queue_find_free(theap,pq); + } + if (page==NULL) return NULL; + mi_assert_internal(mi_page_block_size(page) >= size); + mi_assert_internal(mi_page_immediate_available(page)); + mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(mi_ptr_page(mi_page_start(page))==page); + return page; +} + + + +/* ----------------------------------------------------------- + Users can register a deferred free function called + when the `free` list is empty. Since the `local_free` + is separate this is deterministically called after + a certain number of allocations. +----------------------------------------------------------- */ + +// The program should only install a single deferred free handler before doing allocation. +static _Atomic(void*) deferred_free; // is `mi_deferred_free_fun*` (but some platforms don't support atomic function pointers) +static _Atomic(void*) deferred_arg; + +void _mi_deferred_free(mi_theap_t* theap, bool force) { + theap->heartbeat++; + mi_deferred_free_fun* const fun = (mi_deferred_free_fun*)mi_atomic_load_ptr_acquire(void,&deferred_free); + if (fun != NULL && !theap->tld->recurse) { + theap->tld->recurse = true; + void* const arg = mi_atomic_load_ptr_acquire(void,&deferred_arg); + fun(force, theap->heartbeat, arg); + theap->tld->recurse = false; + } +} + +void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noexcept { + mi_atomic_store_ptr_release(void,&deferred_arg, arg); + mi_atomic_store_ptr_release(void,&deferred_free, (void*)fn); +} + + +/* ----------------------------------------------------------- + Admin +----------------------------------------------------------- */ +static mi_theap_t* mi_theap_init(mi_theap_t* theap) { + if mi_likely(mi_theap_is_initialized(theap)) return theap; + if (theap==&_mi_theap_empty_wrong) { + // we were unable to allocate a theap for a first-class heap + return NULL; + } + // otherwise we initialize the thread and its default theap + theap = _mi_thread_init(); + if mi_unlikely(!mi_theap_is_initialized(theap)) { return NULL; } + mi_assert_internal(mi_theap_is_initialized(theap)); + return theap; +} + +static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap) +{ + theap = mi_theap_init(theap); + if (theap==NULL) return NULL; + mi_assert_internal(mi_theap_is_initialized(theap)); + + // do administrative tasks every N generic mallocs + if mi_unlikely(theap->generic_count >= 1000) { + theap->generic_collect_count += theap->generic_count; + theap->generic_count = 0; + + // check if the profiler is enabled (unless this theap is permanently excluded, see `mi_theap_profile_never`) + if (!theap->profile_never) { + mi_heap_t* const heap = mi_theap_heap(theap); + mi_profiler_t* prof = mi_atomic_load_ptr_relaxed(mi_profiler_t, &heap->profiler); + const bool prof_enabled = (prof!=NULL && mi_profiler_is_enabled(prof)); + if (prof_enabled && theap->profile_sample_rate==0) { + _mi_theap_set_profile_sample_rate(theap,mi_max(1,prof->initial_sample_rate)); // start profiling + } + else if (!prof_enabled && theap->profile_sample_rate!=0) { + _mi_theap_set_profile_sample_rate(theap,0); // stop profiling + } + } + + // do a full theap collect every once in a while (10000 by default) + const long generic_collect = mi_option_get_clamp(mi_option_generic_collect, 1, 1000000L); + if (theap->generic_collect_count >= generic_collect) { + theap->generic_collect_count = 0; + mi_theap_collect(theap, false /* force? */); + } + // else if (theap->sample_rate != 0) { // update stats more aggressively if we are sampling + // mi_theap_collect(theap, false /* force? */); // TODO: make specialized mi_theap_collect_update_stats ? + // } + else { + // otherwise we do a mini-collect + _mi_deferred_free(theap, false); // call potential deferred free routines + _mi_theap_collect_retired(theap, false); // free retired pages + } + } + return theap; +} + +/* ----------------------------------------------------------- + Generic allocation +----------------------------------------------------------- */ + +static bool mi_malloc_generic_check_size(size_t size) { + const size_t req_size = size - MI_PADDING_SIZE; + if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { + _mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size); + return false; + } + return true; +} + +// Allocate from a page of the right size (without sampling) +static void* mi_malloc_generic_from_page(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage, size_t sample_countdown_adjust) +{ + // find (or allocate) a page of the right size + mi_page_t* page = mi_find_page(theap, size, huge_alignment); + if mi_unlikely(page == NULL) { // first time out of memory, try to collect and retry the allocation once more + mi_theap_collect(theap, true /* force? */); + page = mi_find_page(theap, size, huge_alignment); + } + + if mi_unlikely(page == NULL) { // out of memory + theap->sample_countdown -= sample_countdown_adjust; // unadjust again if we failed to find a page + _mi_error_message(ENOMEM, "unable to allocate memory (%zu bytes)\n", size - MI_PADDING_SIZE); + return NULL; + } + + mi_assert_internal(mi_page_immediate_available(page)); + mi_assert_internal(mi_page_block_size(page) >= size); + mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(mi_ptr_page(mi_page_start(page))==page); + + // and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc_zero) + if (ppage!=NULL) { *ppage = page; } + void* const p = _mi_page_malloc_zero(theap,page,size,zero); + mi_assert_internal(p != NULL); + _mi_page_update_sample_countdown(page); + + // move full pages to the full queue + // this will also call _mi_page_update_stats for huge pages + if (mi_page_block_size(page) > MI_SMALL_MAX_OBJ_SIZE) { + if (mi_page_is_full(page)) { + _mi_page_to_full(page, mi_page_queue_of(page)); + } + } + return p; +} + +// Called when the sample countdown is reached: take a sample if possible, or otherwise allocate from a page +static mi_decl_noinline void* mi_malloc_generic_sample(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) +{ + const size_t req_size = size - MI_PADDING_SIZE; + if (huge_alignment==0 && theap->sample_rate!=0) { + return _mi_theap_malloc_sampled(theap,req_size,zero,ppage); + } + mi_assert_internal(!_mi_is_empty_theap(theap)); // cannot write to the empty theap + mi_assert_internal(req_size <= MI_SAMPLE_COUNTDOWN_MAX); + size_t sample_countdown_adjust = 0; + if (theap->sample_rate==0) { + theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // reset the countdown counter + } + #if MI_SAMPLE==2 + else { + // huge_alignment!=0 so we need to adjust the countdown + mi_assert_internal(huge_alignment!=0); + mi_assert_internal(theap->sample_countdown <= SIZE_MAX - req_size); + sample_countdown_adjust = req_size; + theap->sample_countdown += req_size; // adjust such that after `_mi_page_malloc_zero` the countdown is correct again + } + #endif + return mi_malloc_generic_from_page(theap, size, zero, huge_alignment, ppage, sample_countdown_adjust); +} + +static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) +{ + // initialize if necessary + theap = mi_malloc_generic_admin(theap); + if (theap==NULL) return NULL; + + // check allocation size + if (!mi_malloc_generic_check_size(size)) return NULL; + + if mi_unlikely(mi_theap_should_sample(theap, size - MI_PADDING_SIZE)) { + // sampled allocation + return mi_malloc_generic_sample(theap, size, zero, huge_alignment, ppage); + } + else { + // direct allocation + return mi_malloc_generic_from_page(theap, size, zero, huge_alignment, ppage, 0); + } +} + + +// Generic allocation routine if the fast path (`alloc/alloc.c:mi_page_malloc`) does not succeed. +// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed. +// The `huge_alignment` is normally 0 but is set to a multiple of MI_SLICE_SIZE for +// very large requested alignments in which case we use a huge singleton page. +// Note: we put `bool zero, size_t huge_alignment` into one parameter (with zero in bit 0) +// to use 4 parameters which compiles better on msvc for the malloc fast path. +mi_decl_restrict void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept +{ + #if !MI_THEAP_INITASNULL + mi_assert_internal(theap != NULL); + #endif + const bool zero = ((zero_huge_alignment & 1) != 0); + const size_t huge_alignment = (zero_huge_alignment & ~1); + mi_assert_internal(huge_alignment==0 || huge_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN); + mi_page_t* page = NULL; + + // fast path objects that fit in a small page + if mi_likely(mi_theap_is_initialized(theap) && ++theap->generic_count < 1000 && huge_alignment==0) { + const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` + if (req_size < MI_SMALL_MAX_OBJ_SIZE) + { + #if MI_SAMPLE + if mi_likely(!mi_theap_should_sample(theap,req_size)) // ensure we don't need to take a sample + #endif + { + mi_page_queue_t* pq = mi_page_queue(theap, size); + mi_assert_internal(pq!=NULL && !mi_page_queue_is_huge(pq)); + page = mi_page_queue_find_free(theap,pq); + // mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); + if (page!=NULL) { + if (ppage!=NULL) { *ppage = page; } + mi_assert_internal(mi_page_immediate_available(page)); // we should never recurse in _mi_page_malloc_zero + void* p = _mi_page_malloc_zero(theap,page,size,zero); // always succeeds + mi_assert_internal(p != NULL); + return p; + } + } + } + } + // otherwise fallback + return mi_malloc_generic_fallback(theap,size,zero,huge_alignment,ppage); +} + +// Generic allocation without sampling +mi_decl_restrict void* _mi_malloc_generic_no_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept +{ + // do the admin first as it can update the sample rate; we never call it (or the sampling path) recursively here + theap = mi_malloc_generic_admin(theap); + if (theap==NULL) return NULL; + if (!mi_malloc_generic_check_size(size)) return NULL; + const size_t sample_countdown = theap->sample_countdown; + theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // don't count this allocation + void* p = mi_malloc_generic_from_page(theap, size, zero, 0, ppage, 0); + // the sample rate may have changed (e.g. by a profiler `on_free` callback during a collect) + theap->sample_countdown = (theap->sample_rate!=0 && sample_countdown > theap->sample_rate ? theap->sample_rate : sample_countdown); + return p; +} diff --git a/system/lib/mimalloc/src/alloc-override.c b/system/lib/mimalloc/src/alloc/alloc-override.c similarity index 95% rename from system/lib/mimalloc/src/alloc-override.c rename to system/lib/mimalloc/src/alloc/alloc-override.c index 5e8733d4dc6a2..8069644fb09c3 100644 --- a/system/lib/mimalloc/src/alloc-override.c +++ b/system/lib/mimalloc/src/alloc/alloc-override.c @@ -5,10 +5,33 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ +// ------------------------------------------------------ +// Overrides for standard allocation functions +// ------------------------------------------------------ +#include "mimalloc.h" +#include "mimalloc/types.h" + #if !defined(MI_IN_ALLOC_C) #error "this file should be included from 'alloc.c' (so aliases can work)" #endif +#include "mimalloc/track.h" + +// ------------------------------------------------------ +// Defined here instead of alloc-posix.c so they can be aliased below, +// (but still outside of MI_MALLOC_OVERRIDE as they are part of the API.) +// ------------------------------------------------------ + +mi_decl_nodiscard size_t mi_malloc_size(const void* p) mi_attr_noexcept { + if (!mi_any_heap_contains(p)) return 0; + return mi_usable_size(p); +} + +mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept { + if (!mi_any_heap_contains(p)) return 0; + return mi_usable_size(p); +} + #if defined(MI_MALLOC_OVERRIDE) && !defined(_DLL) @@ -61,15 +84,6 @@ typedef void* mi_nothrow_t; // use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1` // See: - struct mi_interpose_s { - const void* replacement; - const void* target; - }; - #define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun } - #define MI_INTERPOSE_MI(fun) MI_INTERPOSE_FUN(fun,mi_##fun) - - #define MI_INTERPOSE_DECLS(name) __attribute__((used)) static struct mi_interpose_s name[] __attribute__((section("__DATA, __interpose"))) - MI_INTERPOSE_DECLS(_mi_interposes) = { MI_INTERPOSE_MI(malloc), @@ -113,7 +127,7 @@ typedef void* mi_nothrow_t; #ifdef __cplusplus } #endif - __attribute__((used)) static struct mi_interpose_s _mi_cxx_interposes[] __attribute__((section("__DATA, __interpose"))) = + MI_INTERPOSE_DECLS(_mi_cxx_interposes) = { MI_INTERPOSE_FUN(_ZdlPv,mi_free), MI_INTERPOSE_FUN(_ZdaPv,mi_free), @@ -130,19 +144,19 @@ typedef void* mi_nothrow_t; void* __cdecl _expand(_Pre_notnull_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size) { return mi__expand(_Block, _Size); } - _Check_return_ _ACRTIMP + _Check_return_ _ACRTIMP size_t __cdecl _msize_base(_Pre_notnull_ void* _Block) _CRT_NOEXCEPT { return mi_malloc_size(_Block); } - _Check_return_ _ACRTIMP _CRT_HYBRIDPATCHABLE + _Check_return_ _ACRTIMP _CRT_HYBRIDPATCHABLE size_t __cdecl _msize(_Pre_notnull_ void* _Block) { return mi_malloc_size(_Block); } - _ACRTIMP + _ACRTIMP void __cdecl _free_base(_Pre_maybenull_ _Post_invalid_ void* _Block) { mi_free(_Block); } - _ACRTIMP _CRT_HYBRIDPATCHABLE + _ACRTIMP _CRT_HYBRIDPATCHABLE void __cdecl free(_Pre_maybenull_ _Post_invalid_ void* _Block) { mi_free(_Block); } @@ -162,7 +176,7 @@ typedef void* mi_nothrow_t; void* __cdecl realloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size) { return mi_realloc(_Block, _Size); } - _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT + _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT void* __cdecl _calloc_base(_In_ size_t _Count, _In_ size_t _Size) { return mi_calloc(_Count, _Size); } @@ -178,7 +192,7 @@ typedef void* mi_nothrow_t; void* __cdecl _recalloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size) { return mi_recalloc(_Block, _Count, _Size); } - _ACRTIMP + _ACRTIMP void __cdecl _aligned_free(_Pre_maybenull_ _Post_invalid_ void* _Block) { mi_free(_Block); } @@ -190,7 +204,7 @@ typedef void* mi_nothrow_t; void* __cdecl _aligned_offset_malloc(_In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment, _In_ size_t _Offset) { return mi_malloc_aligned_at(_Size, _Alignment, _Offset); } - _Check_return_ _ACRTIMP + _Check_return_ _ACRTIMP size_t __cdecl _aligned_msize(_Pre_notnull_ void* _Block, _In_ size_t _Alignment, _In_ size_t _Offset) { MI_UNUSED(_Alignment); MI_UNUSED(_Offset); return mi_malloc_size(_Block); } @@ -226,7 +240,8 @@ typedef void* mi_nothrow_t; #endif #endif -#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) +// on macOS export the C++ operators too: interposing them does not work, but as weak definitions an exported one wins. +#if (defined(__GNUC__) || defined(__clang__)) #pragma GCC visibility push(default) #endif @@ -325,17 +340,6 @@ typedef void* mi_nothrow_t; extern "C" { #endif -// defined here instead alloc-posix so we can alias it -mi_decl_nodiscard size_t mi_malloc_size(const void* p) mi_attr_noexcept { - if (!mi_is_in_heap_region(p)) return 0; - return mi_usable_size(p); -} - -mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept { - if (!mi_is_in_heap_region(p)) return 0; - return mi_usable_size(p); -} - #ifndef MI_OSX_IS_INTERPOSED // Forward Posix/Unix calls as well void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize) @@ -415,7 +419,7 @@ mi_decl_weak int reallocarr(void* p, size_t count, size_t size) { return mi_r } #endif -#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) +#if (defined(__GNUC__) || defined(__clang__)) #pragma GCC visibility pop #endif diff --git a/system/lib/mimalloc/src/alloc-posix.c b/system/lib/mimalloc/src/alloc/alloc-posix.c similarity index 92% rename from system/lib/mimalloc/src/alloc-posix.c rename to system/lib/mimalloc/src/alloc/alloc-posix.c index a2252494255af..4ba1ccb0a10a1 100644 --- a/system/lib/mimalloc/src/alloc-posix.c +++ b/system/lib/mimalloc/src/alloc/alloc-posix.c @@ -10,14 +10,15 @@ terms of the MIT license. A copy of the license can be found in the file // for convenience and used when overriding these functions. // ------------------------------------------------------------------------ #include "mimalloc.h" -#include "mimalloc/internal.h" +#include "mimalloc/types.h" +#include "mimalloc/prim/os.h" +#include "mimalloc/alloc/alloc.h" // ------------------------------------------------------ // Posix & Unix functions definitions // ------------------------------------------------------ #include -#include // memset #include // getenv #ifdef _MSC_VER @@ -40,17 +41,17 @@ int mi_posix_memalign(void** p, size_t alignment, size_t size) { // mi_attr_noe // if (p == NULL) return EINVAL; // it is required that alignment is a power of 2 and a multiple of sizeof(void*) - if (alignment= SIZE_MAX - psize) return NULL; // overflow - size_t asize = _mi_align_up(size, psize); + size_t asize = mi_align_up(size, psize); return mi_malloc_aligned(asize, psize); } @@ -77,7 +78,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size */ // C11 also requires alignment to be a power-of-two (and > 0) which is checked in mi_malloc_aligned void* p = mi_malloc_aligned(size, alignment); - mi_assert_internal(_mi_is_aligned(p,alignment)); + mi_assert_internal(mi_is_aligned(p,alignment)); return p; } @@ -126,11 +127,11 @@ mi_decl_nodiscard mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s) mi_attr_ if (s==NULL) return NULL; size_t wlen; for(wlen = 0; s[wlen] != 0 && wlen < PTRDIFF_MAX; wlen++) { } // prevent overflow on wlen+1 - size_t size; + size_t size; if (mi_mul_overflow(wlen+1, sizeof(wchar_t), &size) || size > PTRDIFF_MAX) return NULL; wchar_t* p = (wchar_t*)mi_malloc(size); if (p != NULL) { - _mi_memcpy(p,s,size); + mi_memcpy(p,s,size); } return p; } @@ -156,7 +157,7 @@ int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept { int mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name) mi_attr_noexcept { if (size != NULL) *size = 0; - if (buf==NULL || name==NULL) return EINVAL; + if (buf==NULL || name==NULL) return EINVAL; #if !defined(_WIN32) || (defined(WINAPI_FAMILY) && (WINAPI_FAMILY != WINAPI_FAMILY_DESKTOP_APP)) // not supported *buf = NULL; diff --git a/system/lib/mimalloc/src/alloc.c b/system/lib/mimalloc/src/alloc/alloc.c similarity index 80% rename from system/lib/mimalloc/src/alloc.c rename to system/lib/mimalloc/src/alloc/alloc.c index a176db77f41a6..f3ea5e08eb484 100644 --- a/system/lib/mimalloc/src/alloc.c +++ b/system/lib/mimalloc/src/alloc/alloc.c @@ -8,13 +8,15 @@ terms of the MIT license. A copy of the license can be found in the file #define _DEFAULT_SOURCE // for realpath() on Linux #endif -#include "mimalloc.h" -#include "mimalloc/internal.h" -#include "mimalloc/atomic.h" -#include "mimalloc/prim-tls.h" // _mi_prim_thread_id() +#include // malloc, abort, realpath -#include // memset, strlen (for mi_strdup) -#include // malloc, abort +#include "mimalloc.h" +#include "mimalloc/types.h" +#include "mimalloc/prim/prim-tls.h" // _mi_prim_thread_id() +#include "mimalloc/util/stats.h" +#include "mimalloc/alloc/alloc.h" +#include "mimalloc/alloc/free.h" +#include "mimalloc/profile/sample-profile.h" #define MI_IN_ALLOC_C #include "alloc-override.c" @@ -24,7 +26,7 @@ terms of the MIT license. A copy of the license can be found in the file // ------------------------------------------------------ // Core allocation functions. -// Every allocation goes through `mi_page_malloc_zero` +// Every allocation goes through `mi_page_malloc_zero` // ------------------------------------------------------ #if MI_PADDING @@ -46,46 +48,75 @@ static mi_decl_noinline void mi_page_block_setup_padding(mi_page_t* page, mi_blo } #endif +// C++ new handler support +static void* mi_theap_new_handler(mi_theap_t* theap, size_t size); +static mi_decl_noinline void* mi_malloc_generic_new(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) { + void* p = _mi_malloc_generic(theap, size, zero_huge_alignment, ppage); + if (p == NULL) { return mi_theap_new_handler(theap, size - MI_PADDING_SIZE); } + return p; +} +#if defined(_MSC_VER) && !MI_CLANG_CL +// for the msvc compiler to get tail-calls +static mi_decl_noinline void* mi_malloc_generic_msvc(mi_theap_t* theap, size_t size, size_t zero_new_huge_alignment, mi_page_t** ppage) mi_attr_noexcept { + const bool is_new = ((zero_new_huge_alignment & 2) != 0); + const size_t zero_huge_alignment = (zero_new_huge_alignment & ~2); + if (is_new) { return mi_malloc_generic_new(theap, size, zero_huge_alignment, ppage); } + else { return _mi_malloc_generic(theap, size, zero_huge_alignment, ppage); } +} +#endif + +// C++ new calls a new-handler on failure. +#if MI_SAMPLE==2 +static mi_decl_noinline mi_decl_restrict void* mi_theap_malloc_sampled_new(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage); +#endif + // Fast allocation in a page: just pop from the free list. // Fall back to generic allocation only if the list is empty. // Note: even though there is a lot of checks etc in the source, // in release mode the (inlined) routine is about 7 to 10 instructions with a single test. -static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, size_t sample_countdown, bool zero, mi_page_t** ppage) mi_attr_noexcept +static mi_decl_forceinline mi_decl_restrict void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, size_t sample_countdown, bool zero, bool is_new, mi_page_t** ppage) { if (page->block_size != 0) { // not the empty theap mi_assert_internal(mi_page_block_size(page) >= size); - mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); - mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); - mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); } // check the free list mi_block_t* const block = page->free; - mi_used_t xused = page->xused; - xused.used_alloc += 0x10001; // increment both (16-bit) used count and alloc count - #if defined(__GNUC__) - __asm("" : : : "memory" ); // always load the `used` field before the test - #endif - if (block == NULL) { - return _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); - } - mi_assert_internal(block != NULL && _mi_ptr_page(block) == page); + mi_used_t xused = page->xused; + #if defined(__GNUC__) + __asm("" : : "r"(xused) : ); // load the `xused` field before the test + #endif + xused.used_alloc += 0x10001; + if mi_unlikely(block == NULL) { + #if defined(_MSC_VER) && !MI_CLANG_CL + return mi_malloc_generic_msvc(theap, size, (zero ? 1 : 0) | (is_new ? 2 : 0), ppage); + #else + if (is_new) { return mi_malloc_generic_new(theap, size, (zero ? 1 : 0), ppage); } + else { return _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); } + #endif + } + mi_assert_internal(block != NULL && mi_ptr_page(block) == page); if (ppage != NULL) { *ppage = page; }; // pop from the free list mi_block_t* next = mi_block_next(page,block); mi_track_mem_undefined(block,sizeof(*block)); - + #if MI_SECURE if (!zero) block->next = 0; // don't leak internal data #endif - page->free = next; + // update the page free list and used count + page->free = next; page->xused = xused; - mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page); - mi_assert_internal(page->block_size < MI_MAX_ALIGN_SIZE || _mi_is_aligned(block, MI_MAX_ALIGN_SIZE)); - #if MI_SAMPLE==2 + mi_assert_internal(page->free == NULL || mi_ptr_page(page->free) == page); + mi_assert_internal(page->block_size < MI_MAX_ALIGN_SIZE || mi_is_aligned(block, MI_MAX_ALIGN_SIZE)); + + #if MI_SAMPLE==2 const size_t req_size = size - MI_PADDING_SIZE; mi_assert_internal(theap->sample_countdown == sample_countdown); // we pass it to improve codegen mi_assert_internal(theap->sample_countdown >= req_size); @@ -112,10 +143,10 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ // track per-block statistics mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); - + // in debug mode initialize with 0xD0 #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN - if mi_likely(!zero && !mi_page_is_huge(page)) { memset(block, MI_DEBUG_UNINIT, bsize); } + if mi_likely(!zero && !mi_page_is_huge(page)) { mi_debug_fill(page, block, MI_DEBUG_UNINIT, bsize); } #endif // zero the block? note: we need to zero the full block size (issue #63) @@ -123,34 +154,34 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ if (!page->free_is_zero) { mi_assert_internal(bsize%MI_SIZE_SIZE == 0); #if !MI_PADDING - return // use tail-call + return // use tail-call #endif - _mi_memzero_block(block,bsize); + mi_memzero_block(block,bsize); } else { - block->next = 0; - mi_track_mem_defined(block, bsize); + block->next = 0; + mi_track_mem_defined(block, bsize); } } - + // setup padding (must come after zero'ing) #if MI_PADDING // && !MI_TRACK_ENABLED mi_page_block_setup_padding(page,block,size); #endif - return block; + return block; } -// extra entries for improved efficiency in `alloc-aligned.c` (and in `page.c:mi_malloc_generic`. -extern void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept { - return mi_page_malloc_zero(theap, page, size, theap->sample_countdown, zero, NULL); +// extra entries for improved efficiency in `alloc/alloc-aligned.c` (and in `alloc/alloc-generic.c:mi_malloc_generic`. +extern mi_decl_restrict void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept { + return mi_page_malloc_zero(theap, page, size, theap->sample_countdown, zero, false, NULL); } // Internal small size allocation; assumes the `theap` is non-NULL. // This looks up the page directly from a the direct page entries in the theap. // The size can be in words (as a wsize); we assume this will be inlined for best code for `mi_wmalloc_small` and similar functions. -static mi_decl_forceinline mi_decl_restrict void* mi_theap_nonnull_xmalloc_small_zero(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept +static mi_decl_forceinline mi_decl_restrict void* mi_theap_nonnull_xmalloc_small_zero(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, bool is_new, mi_page_t** ppage) { size_t size = (is_wsize ? xsize * MI_SIZE_SIZE : xsize); mi_assert(theap != NULL); @@ -161,17 +192,20 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_nonnull_xmalloc_small #if (MI_PADDING) // || MI_GUARDED) if mi_unlikely(xsize == 0) { xsize = (is_wsize ? 1 : MI_SIZE_SIZE); size += MI_SIZE_SIZE; } #endif - + // we only sample if fine-grained sampling is enabled (otherwise we sample in mi_malloc_generic) - #if MI_SAMPLE==2 - if mi_unlikely(mi_theap_should_sample(theap,size)) { return _mi_theap_malloc_sampled(theap,size,zero,ppage); } + #if MI_SAMPLE==2 + if mi_unlikely(mi_theap_should_sample(theap,size)) { + if (is_new) { return mi_theap_malloc_sampled_new(theap,size,zero,ppage); } + else { return _mi_theap_malloc_sampled(theap,size,zero,ppage); } + } #endif - - // get page in constant time - mi_page_t* page = _mi_theap_get_free_small_page(theap, xsize + (is_wsize ? MI_PADDING_WSIZE : MI_PADDING_SIZE), is_wsize); - // and allocate - void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, theap->sample_countdown, zero, ppage); + // get page in constant time + mi_page_t* page = mi_theap_get_free_small_page(theap, xsize + (is_wsize ? MI_PADDING_WSIZE : MI_PADDING_SIZE), is_wsize); + + // and allocate + void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, theap->sample_countdown, zero, is_new, ppage); mi_track_malloc(p,size,zero); #if MI_DEBUG>3 @@ -183,7 +217,7 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_nonnull_xmalloc_small } // Internal generic allocation. -// Goes to `page.c:_mi_malloc_generic` which handles large and aligned allocations. +// Goes to `alloc/alloc-generic.c:_mi_malloc_generic` which handles large and aligned allocations. // Every small allocation eventually goes through this generic allocation path if it cannot be satisfied from the small page cache. static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { @@ -210,10 +244,10 @@ static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size // internal small allocation where the theap can be NULL on some platforms (Windows and macOS) static mi_decl_forceinline mi_decl_restrict void* mi_theap_xmalloc_small_zero(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept { #if !MI_THEAP_INITASNULL - return mi_theap_nonnull_xmalloc_small_zero(theap, xsize, is_wsize, zero, ppage); + return mi_theap_nonnull_xmalloc_small_zero(theap, xsize, is_wsize, zero, false, ppage); #else if (theap!=NULL) { - return mi_theap_nonnull_xmalloc_small_zero(theap, xsize, is_wsize, zero, ppage); + return mi_theap_nonnull_xmalloc_small_zero(theap, xsize, is_wsize, zero, false, ppage); } else { return mi_theap_malloc_generic(theap, (is_wsize ? xsize * MI_SIZE_SIZE : xsize), zero, 0, ppage); // tailcall @@ -224,22 +258,22 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_xmalloc_small_zero(mi // Allocate a small block from a theap; these routines assume a non-NULL theap. mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { mi_assert(theap!=NULL); - return mi_theap_nonnull_xmalloc_small_zero(theap, size, false, false, NULL); + return mi_theap_nonnull_xmalloc_small_zero(theap, size, false, false, false, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept { mi_assert(theap!=NULL); - return mi_theap_nonnull_xmalloc_small_zero(theap, wsize, true, false, NULL); + return mi_theap_nonnull_xmalloc_small_zero(theap, wsize, true, false, false, NULL); } mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { mi_assert(theap!=NULL); - return mi_theap_nonnull_xmalloc_small_zero(theap, size, false, true, NULL); + return mi_theap_nonnull_xmalloc_small_zero(theap, size, false, true, false, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_theap_wzalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept { mi_assert(theap!=NULL); - return mi_theap_nonnull_xmalloc_small_zero(theap, wsize, true, true, NULL); + return mi_theap_nonnull_xmalloc_small_zero(theap, wsize, true, true, false, NULL); } // Regular small allocation functions that use the default theap. @@ -252,7 +286,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_no } mi_decl_nodiscard mi_decl_restrict void* mi_wmalloc_small(size_t wsize) mi_attr_noexcept { - return mi_theap_xmalloc_small_zero(_mi_theap_default(), wsize, true, false, NULL ); + return mi_theap_xmalloc_small_zero(_mi_theap_default(), wsize, true, false, NULL ); } mi_decl_nodiscard mi_decl_restrict void* mi_wzalloc_small(size_t wsize) mi_attr_noexcept { @@ -268,7 +302,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, s // but the following prevents using a stack frame. We use this to optimize some select functions only. mi_theap_t* const theap = _mi_heap_theap_cached(heap); if mi_likely(theap!=NULL) { return mi_theap_malloc_small(theap, size); } - else { return mi_heap_init_malloc_small(heap, size); } + else { return mi_heap_init_malloc_small(heap, size); } } mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { @@ -278,7 +312,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, s // optimize: return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, true, NULL); mi_theap_t* const theap = _mi_heap_theap_cached(heap); if mi_likely(theap!=NULL) { return mi_theap_zalloc_small(theap, size); } - else { return mi_heap_init_zalloc_small(heap, size); } + else { return mi_heap_init_zalloc_small(heap, size); } } @@ -291,7 +325,7 @@ mi_decl_nodiscard static mi_decl_forceinline mi_decl_restrict void* mi_theap_non // fast path for small objects if mi_likely(size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(huge_alignment == 0); - return mi_theap_nonnull_xmalloc_small_zero(theap, size, false, zero, ppage); + return mi_theap_nonnull_xmalloc_small_zero(theap, size, false, zero, false, ppage); } else { return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage); @@ -307,7 +341,7 @@ mi_decl_nodiscard extern mi_decl_forceinline mi_decl_restrict void* _mi_theap_ma #endif { mi_assert_internal(huge_alignment == 0); - return mi_theap_nonnull_xmalloc_small_zero(theap, size, false, zero, ppage); + return mi_theap_nonnull_xmalloc_small_zero(theap, size, false, zero, false, ppage); } else { return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage); @@ -350,7 +384,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t // return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, false, 0, NULL); mi_theap_t* const theap = _mi_heap_theap_cached(heap); if mi_likely(theap!=NULL) { return mi_theap_nonnull_malloc(theap, size); } - else { return mi_heap_init_malloc(heap, size); } + else { return mi_heap_init_malloc(heap, size); } } @@ -361,7 +395,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t // optimize: return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, true, 0, NULL); mi_theap_t* const theap = _mi_heap_theap_cached(heap); if mi_likely(theap!=NULL) { return mi_theap_nonnull_zalloc(theap, size); } - else { return mi_heap_init_zalloc(heap, size); } + else { return mi_heap_init_zalloc(heap, size); } } // ---------------------------------------------------------- @@ -388,14 +422,14 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t // Return usable size static void* mi_ublock_size( void* p, mi_page_t* page, size_t* pblock_size ) { - mi_assert_internal(page == _mi_ptr_page(p)); + mi_assert_internal(page == mi_ptr_page(p)); if (pblock_size!=NULL) { if (p!=NULL) { *pblock_size = mi_page_block_size(page); } } return p; } -mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; void* p = mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, false, &page); return mi_ublock_size(p,page,pblock_size); @@ -453,7 +487,7 @@ void* mi_expand(void* p, size_t newsize) mi_attr_noexcept { return NULL; #else if (p == NULL) return NULL; - const mi_page_t* const page = mi_ptr_page_validate(p,"mi_expand"); + const mi_page_t* const page = mi_ptr_page_validate(p,"mi_expand"); const size_t size = _mi_page_usable_size(page,p); if (newsize > size) return NULL; return p; // it fits @@ -471,24 +505,24 @@ static mi_decl_forceinline void* mi_theap_realloc_zero_ex(mi_theap_t* theap, voi size = 0; if (pblock_size_pre!=NULL) { *pblock_size_pre = 0; } } - else { - page = mi_ptr_page_validate(p,"mi_realloc"); + else { + page = mi_ptr_page_validate(p,"mi_realloc"); if mi_unlikely(page==NULL) { // invalid pointer if (pblock_size_pre!=NULL) { *pblock_size_pre = 0; } - if (pblock_size_post!=NULL) { *pblock_size_post = 0; } + if (pblock_size_post!=NULL) { *pblock_size_post = 0; } return NULL; - } + } size = _mi_page_usable_size(page,p); if (pblock_size_pre!=NULL) { *pblock_size_pre = mi_page_block_size(page); } } // check if we can reuse the existing block - if mi_unlikely(newsize<=size && newsize>=(size/2) && newsize>0) { // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0) + if mi_unlikely(newsize<=size && newsize>=(size/2) && newsize>0) { // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0) mi_assert_internal(page!=NULL); // note: page!=NULL (since if p==NULL, we have size=0 and size>=newsize>0 #if MI_THEAP_INITASNULL if (theap!=NULL) #endif { - if (mi_page_heap(page)==_mi_theap_heap_peek(theap)) { // and within the same heap + if (mi_page_heap(page)==mi_theap_heap_peek(theap)) { // and within the same heap mi_assert_internal(p!=NULL); // todo: do not track as the usable size is still the same in the free; adjust potential padding? // mi_track_resize(p,size,newsize) @@ -498,24 +532,24 @@ static mi_decl_forceinline void* mi_theap_realloc_zero_ex(mi_theap_t* theap, voi } } } - // note: we don't zero allocate upfront so we only zero initialize the expanded part + // note: we don't zero allocate upfront so we only zero initialize the expanded part mi_page_t* newpage; // use block_size for zero-ing, issue #763 - void* const newp = _mi_theap_malloc_zero(theap,newsize,false /* no zero */,0,&newpage); + void* const newp = _mi_theap_malloc_zero(theap,newsize,false /* no zero */,0,&newpage); if mi_likely(newp != NULL) { - if (pblock_size_post!=NULL) { *pblock_size_post = mi_page_block_size(newpage); } + if (pblock_size_post!=NULL) { *pblock_size_post = mi_page_block_size(newpage); } const size_t copy_size = (newsize > size ? size : newsize); - const size_t zero_start = _mi_align_down( (copy_size >= sizeof(intptr_t) ? copy_size - sizeof(intptr_t) : 0), sizeof(intptr_t)); // also set last word in the previous allocation to zero to ensure any padding is zero-initialized - const size_t usable = _mi_page_usable_size(newpage,newp); - mi_assert_internal(usable >= newsize); - if (zero && usable > zero_start) { - _mi_memzero_aligned((uint8_t*)newp + zero_start, usable - zero_start); + const size_t zero_start = mi_align_down( (copy_size >= sizeof(intptr_t) ? copy_size - sizeof(intptr_t) : 0), sizeof(intptr_t)); // also set last word in the previous allocation to zero to ensure any padding is zero-initialized + const size_t usable = _mi_page_usable_size(newpage,newp); + mi_assert_internal(usable >= newsize); + if (zero && usable > zero_start) { + mi_memzero_aligned((uint8_t*)newp + zero_start, usable - zero_start); } else if (newsize == 0) { ((uint8_t*)newp)[0] = 0; // work around for applications that expect zero-reallocation to be zero initialized (issue #725) } if mi_likely(p != NULL) { mi_track_mem_defined(p,copy_size); // _mi_useable_size may be too large for byte precise memory tracking.. - _mi_memcpy_aligned(newp, p, copy_size); + mi_memcpy_aligned(newp, p, copy_size); mi_free(p); // only free the original pointer if successful } } @@ -613,13 +647,13 @@ mi_decl_nodiscard void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t count, // ------------------------------------------------------ // `strdup` using mi_malloc -mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strdup(mi_theap_t* xtheap, const char* s) mi_attr_noexcept { +mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strdup(mi_theap_t* xtheap, const char* s) mi_attr_noexcept { if (s == NULL) return NULL; size_t len = _mi_strlen(s); if (len > MI_MAX_ALLOC_SIZE - 1) return NULL; // prevent overflow on len+1 char* t = (char*)mi_theap_malloc(xtheap,len+1); if (t == NULL) return NULL; - _mi_memcpy(t, s, len); + mi_memcpy(t, s, len); t[len] = 0; return t; } @@ -639,7 +673,7 @@ mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strndup(mi_theap_t* xth if (len > MI_MAX_ALLOC_SIZE - 1) return NULL; // prevent overflow on len+1 char* t = (char*)mi_theap_malloc(xtheap, len+1); if (t == NULL) return NULL; - _mi_memcpy(t, s, len); + mi_memcpy(t, s, len); t[len] = 0; return t; } @@ -683,7 +717,7 @@ mi_decl_nodiscard static mi_decl_restrict char* mi_theap_realpath(mi_theap_t* xt return NULL; } else if (res > PATH_MAX) { - errno = ENAMETOOLONG; + errno = ENAMETOOLONG; return NULL; } else if (resolved_name != NULL) { @@ -699,7 +733,7 @@ mi_decl_nodiscard static mi_decl_restrict char* mi_theap_realpath(mi_theap_t* xt #include // pathconf static size_t mi_path_max(void) { - static _Atomic(size_t) path_max = 0; + static _Atomic(size_t) path_max; // = 0 size_t pmax = mi_atomic_load_acquire(&path_max); if (pmax == 0) { long m = 0; @@ -725,9 +759,9 @@ char* mi_theap_realpath(mi_theap_t* xtheap, const char* fname, char* resolved_na char* rname = realpath(fname, NULL); if (rname == NULL) return NULL; char* result = mi_heap_strdup(heap, rname); - mi_cfree(rname); // note: may leak the original pointer if allocated internally with the system allocator + mi_cfree(rname); // note: may leak the original pointer if allocated internally with the system allocator // note: with ASAN realpath is intercepted and mi_cfree may leak the returned pointer :-( - return result; + return result; */ const size_t n = mi_path_max(); char* const buf = (char*)mi_theap_zalloc(xtheap,n+1); @@ -766,7 +800,7 @@ it returns false, but we limit the total calls. #ifdef __cplusplus #include -static bool mi_try_new_handler(bool nothrow) { +static bool mi_try_new_handler(void) { #if defined(_MSC_VER) || (__cplusplus >= 201103L) std::new_handler h = std::get_new_handler(); #else @@ -776,32 +810,21 @@ static bool mi_try_new_handler(bool nothrow) { if (h==NULL) { _mi_error_message(ENOMEM, "out of memory in 'new'"); #if defined(_CPPUNWIND) || defined(__cpp_exceptions) // exceptions are not always enabled - if (!nothrow) { - throw std::bad_alloc(); - } - #else - MI_UNUSED(nothrow); + throw std::bad_alloc(); #endif return false; } - else if (!nothrow) { - h(); - return true; - } else { - try { - h(); - } - catch(...) { // swallow std::bad_alloc - return false; // stop trying - } + h(); return true; } } #else typedef void (*std_new_handler_t)(void); -#if (defined(__GNUC__) || (defined(__clang__) && !defined(_MSC_VER))) // exclude clang-cl, see issue #631 +// Define MI_NO_CXX_NEW_HANDLER to not define a weak `std::get_new_handler` (`_ZSt15get_new_handlerv`) +// when the C++ new-handler is never used (e.g. when only the C API is used). +#if (defined(__GNUC__) || (defined(__clang__) && !defined(_MSC_VER))) && !defined(MI_NO_CXX_NEW_HANDLER) // exclude clang-cl, see issue #631 std_new_handler_t __attribute__((weak)) _ZSt15get_new_handlerv(void) { return NULL; } @@ -815,13 +838,11 @@ static std_new_handler_t mi_get_new_handler(void) { } #endif -static bool mi_try_new_handler(bool nothrow) { +static bool mi_try_new_handler(void) { std_new_handler_t h = mi_get_new_handler(); if (h==NULL) { _mi_error_message(ENOMEM, "out of memory in 'new'"); - if (!nothrow) { - abort(); // cannot throw in plain C, use abort - } + abort(); // cannot throw in plain C, use abort return false; } else { @@ -831,40 +852,64 @@ static bool mi_try_new_handler(bool nothrow) { } #endif -static mi_decl_noinline void* mi_theap_try_new(mi_theap_t* theap, size_t size, bool nothrow ) { +static mi_decl_noinline mi_decl_restrict void* mi_try_new_handler_null(void) { + mi_try_new_handler(); + return NULL; +} + +// called when an allocation fails and the new handler needs to be invoked +static mi_decl_noinline void* mi_theap_new_handler(mi_theap_t* theap, size_t size) { void* p = NULL; - for(int i = 0; i < MI_TRY_NEW_MAX && p == NULL && mi_try_new_handler(nothrow); i++) { + for(int i = 0; i < MI_TRY_NEW_MAX && p == NULL && mi_try_new_handler(); i++) { if (size > MI_MAX_ALLOC_SIZE) return NULL; // call try_new_handler at least once p = mi_theap_malloc(theap,size); } return p; } -static mi_decl_noinline void* mi_heap_try_new(mi_heap_t* heap, size_t size, bool nothrow) { - return mi_theap_try_new(_mi_heap_theap(heap), size, nothrow); +#if MI_SAMPLE==2 +static mi_decl_noinline void* mi_theap_malloc_sampled_new(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage) { + void* p = _mi_theap_malloc_sampled(theap, req_size, zero, ppage); + if mi_unlikely(p == NULL) { + return mi_theap_new_handler(theap, req_size); + } + return p; } +#endif +// Fast tail calling allocation that invokes the new handler on failure (through `mi_malloc_generic_new`) mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* theap, size_t size) { - void* p = mi_theap_malloc(theap,size); - if mi_unlikely(p == NULL) return mi_theap_try_new(theap, size, false); - return p; + // fast path for small objects + #if MI_THEAP_INITASNULL + if mi_likely(theap!=NULL && size <= MI_SMALL_SIZE_MAX) + #else + if mi_likely(size <= MI_SMALL_SIZE_MAX) + #endif + { + return mi_theap_nonnull_xmalloc_small_zero(theap, size, false /* is wsize? */, false /* zero */, true /* is_new */, NULL); + } + else { + return mi_malloc_generic_new(theap, size + MI_PADDING_SIZE, 0, NULL); + } } mi_decl_nodiscard mi_decl_restrict void* mi_new(size_t size) { return mi_theap_alloc_new(_mi_theap_default(), size); } +mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_new(mi_heap_t* heap, size_t size) { + return mi_theap_alloc_new(_mi_heap_theap_get_or_init(heap), size); +} mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) { - void* p = mi_heap_malloc(heap, size); - if mi_unlikely(p == NULL) return mi_heap_try_new(heap, size, false); - return p; + mi_theap_t* const theap = _mi_heap_theap_cached(heap); + if mi_likely(theap!=NULL) { return mi_theap_alloc_new(theap, size); } + else { return mi_heap_init_new(heap, size); } } mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* theap, size_t count, size_t size) { size_t total; if mi_unlikely(mi_count_size_overflow(count, size, &total)) { - mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc - return NULL; + return mi_try_new_handler_null(); // on overflow we tail invoke the try_new_handler once to potentially throw std::bad_alloc } else { return mi_theap_alloc_new(theap,total); @@ -875,24 +920,26 @@ mi_decl_nodiscard mi_decl_restrict void* mi_new_n(size_t count, size_t size) { return mi_theap_alloc_new_n(_mi_theap_default(), count, size); } +mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_new_n(mi_heap_t* heap, size_t count, size_t size) { + return mi_theap_alloc_new_n(_mi_heap_theap_get_or_init(heap), count, size); +} mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) { - return mi_theap_alloc_new_n(_mi_heap_theap(heap), count, size); + mi_theap_t* const theap = _mi_heap_theap_cached(heap); + if mi_likely(theap!=NULL) { return mi_theap_alloc_new_n(theap, count, size); } + else { return mi_heap_init_new_n(heap, count, size); } } - mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new_nothrow(mi_theap_t* theap, size_t size) mi_attr_noexcept { - void* p = mi_theap_malloc(theap,size); - if mi_unlikely(p == NULL) return mi_theap_try_new(theap, size, true); - return p; + return mi_theap_malloc(theap,size); } mi_decl_nodiscard mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept { - return mi_theap_alloc_new_nothrow(_mi_theap_default(), size); + return mi_malloc(size); } -static mi_decl_noinline void* mi_try_new_aligned(size_t size, size_t alignment, bool nothrow) { +static mi_decl_noinline void* mi_try_new_aligned(size_t size, size_t alignment) { void* p = NULL; - for(int i = 0; i < MI_TRY_NEW_MAX && p==NULL && mi_try_new_handler(nothrow); i++) { + for(int i = 0; i < MI_TRY_NEW_MAX && p==NULL && mi_try_new_handler(); i++) { if (!mi_alignment_is_valid(alignment)) return NULL; p = mi_malloc_aligned(size,alignment); } @@ -901,19 +948,17 @@ static mi_decl_noinline void* mi_try_new_aligned(size_t size, size_t alignment, mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) { void* p = mi_malloc_aligned(size, alignment); - if mi_unlikely(p==NULL) return mi_try_new_aligned(size,alignment,false); + if mi_unlikely(p==NULL) return mi_try_new_aligned(size,alignment); return p; } mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept { - void* p = mi_malloc_aligned(size, alignment); - if mi_unlikely(p==NULL) return mi_try_new_aligned(size,alignment,true); - return p; + return mi_malloc_aligned(size, alignment); } static mi_decl_noinline void* mi_try_new_realloc(void* p, size_t newsize) { void* q = NULL; - for(int i = 0; i < MI_TRY_NEW_MAX && q==NULL && mi_try_new_handler(false); i++) { + for(int i = 0; i < MI_TRY_NEW_MAX && q==NULL && mi_try_new_handler(); i++) { if (newsize > MI_MAX_ALLOC_SIZE) return NULL; q = mi_realloc(p,newsize); } @@ -929,8 +974,7 @@ mi_decl_nodiscard void* mi_new_realloc(void* p, size_t newsize) { mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) { size_t total; if mi_unlikely(mi_count_size_overflow(newcount, size, &total)) { - mi_try_new_handler(false); - return NULL; + return mi_try_new_handler_null(); } else { return mi_new_realloc(p, total); diff --git a/system/lib/mimalloc/src/free.c b/system/lib/mimalloc/src/alloc/free.c similarity index 79% rename from system/lib/mimalloc/src/free.c rename to system/lib/mimalloc/src/alloc/free.c index 722ee7bf6933a..4d6f2b4e4a679 100644 --- a/system/lib/mimalloc/src/free.c +++ b/system/lib/mimalloc/src/alloc/free.c @@ -6,18 +6,36 @@ terms of the MIT license. A copy of the license can be found in the file -----------------------------------------------------------------------------*/ #if !defined(MI_IN_ALLOC_C) #error "this file should be included from 'alloc.c' (so aliases can work from alloc-override)" -// add includes help an IDE -#include "mimalloc.h" -#include "mimalloc/internal.h" -#include "mimalloc/prim-tls.h" // _mi_prim_thread_id() #endif +#include "mimalloc.h" +#include "mimalloc/types.h" +#include "mimalloc/prim/prim-tls.h" // _mi_prim_thread_id() +#include "mimalloc/util/stats.h" +#include "mimalloc/arena/arena-page.h" +#include "mimalloc/arena/arena.h" // _mi_arena_memid_is_suitable +#include "mimalloc/profile/sample-guarded.h" // unguard +#include "mimalloc/alloc/page.h" +#include "mimalloc/alloc/free.h" +#include "mimalloc/heap/heap.h" + // forward declarations mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size); mi_decl_nodiscard static bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block); static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool was_guarded); +#if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN +// Fill a block with a debug pattern, preserving the first `heap->debug_fill_preserve` bytes +// (used by CPython to keep object headers intact that may be read by concurrent lock-free readers) +static inline void mi_debug_fill(const mi_page_t* page, void* block, int c, size_t size) { + const size_t preserve = mi_atomic_load_relaxed(&mi_page_heap(page)->debug_fill_preserve); + if (preserve < size) { + mi_memset((uint8_t*)block + preserve, c, size - preserve); + } +} +#endif + // ------------------------------------------------------ // Free // ------------------------------------------------------ @@ -26,29 +44,32 @@ static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* bl // fast path written carefully to prevent spilling on the stack static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool was_guarded, bool check_full) { - // checks + // checks size_t usable_size; - if mi_unlikely(!mi_check_padding_on_free(page, block, was_guarded, &usable_size)) return; + if mi_unlikely(!mi_check_padding_on_free(page, block, was_guarded, &usable_size)) return; if mi_unlikely(!mi_check_double_free(page,block)) return; // usually checked with padding - // mi_stat_free(page, block); - mi_track_free_size(block, usable_size); + // mi_stat_free(page, block); + mi_track_free_size(block, usable_size); #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN const size_t dbgsize = (usable_size > MI_MiB ? MI_MiB : usable_size); - _mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize); + mi_debug_fill(page, block, MI_DEBUG_FREED, dbgsize); #endif - - // actual free: push on the local free list + + // actual free: push on the local free list fast-path mi_used_t xused = page->xused; - xused.used_alloc--; // decrement used count - mi_block_set_next(page, block, page->local_free); + mi_block_t* lfree = page->local_free; + xused.used_alloc--; + mi_block_set_next(page, block, lfree); page->xused = xused; page->local_free = block; + const bool is_empty = (mi_xused_used_count(xused) == 0); mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); - if mi_unlikely(mi_xused_used_count(xused) == 0) { // is used count zero ? + if mi_unlikely(is_empty) { // is used count zero ? + _mi_page_qsbr_clear(page); // a new empty interval must not reuse an earlier grace period if (page->retire_expire==0) { // no need to re-retire retired pages (happens when we alloc/free one block repeatedly in an empty page) - _mi_page_retire(page); + _mi_page_retire(page); } } else if mi_unlikely(check_full && mi_page_is_in_full(page)) { @@ -60,20 +81,22 @@ static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free, bool allow_reclaim) mi_attr_noexcept; // Free a block multi-threaded -static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded, bool allow_reclaim) mi_attr_noexcept +#if defined(_MSC_VER) +static mi_decl_noinline /* ensures mi_free has no stack frame */ +#else +static inline +#endif +void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded, bool allow_reclaim) mi_attr_noexcept { size_t usable_size; if mi_unlikely(!mi_check_padding_on_free(page, block, was_guarded, &usable_size)) return; // checking padding is safe for mt - - // adjust stats (after padding check ) - // mi_stat_free(page, block); // stat_free may access the padding mi_track_free_size(block, usable_size); // _mi_padding_shrink(page, block, sizeof(mi_block_t)); #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threading if (!was_guarded) { const size_t dbgsize = (usable_size > MI_MiB ? MI_MiB : usable_size); - _mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize); + mi_debug_fill(page, block, MI_DEBUG_FREED, dbgsize); } #endif @@ -84,10 +107,11 @@ static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was do { mi_block_set_next(page, block, mi_tf_block(tf_old)); tf_new = mi_tf_create(block, true); - } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new)); + } while(!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new)); // and atomically try to collect the page if it was abandoned - const bool is_newly_owned = mi_tf_is_owned(tf_new) && !mi_tf_is_owned(tf_old); + mi_assert_internal(mi_tf_is_owned(tf_new)); + const bool is_newly_owned = !mi_tf_is_owned(tf_old); // && mi_tf_is_owned(tf_new) if (is_newly_owned) { mi_assert_internal(mi_page_is_abandoned(page)); // mi_assert_internal(!mi_page_is_abandoned_mapped(page)); @@ -105,9 +129,9 @@ static inline mi_block_t* mi_page_ptr_unalign_ex(const mi_page_t* page, const vo const size_t diff = (uint8_t*)p - mi_page_start(page); const size_t block_size = mi_page_block_size(page); - size_t adjust = diff & (block_size - 1); - if mi_unlikely(!_mi_is_power_of_two(block_size)) { - adjust = diff % block_size; + size_t adjust = diff & (block_size - 1); + if mi_unlikely(!mi_is_power_of_two(block_size)) { + adjust = diff % block_size; } if (poffset!=NULL) { *poffset = adjust; } return (mi_block_t*)((uintptr_t)p - adjust); @@ -140,14 +164,14 @@ static inline mi_block_t* mi_page_ptr_block_check(mi_page_t* page, void* p, bool if (offset >= sizeof(mi_block_t)) { #if MI_PROFILE if (block->next == MI_BLOCK_TAG_PROFILED) { - _mi_page_profile_on_free(page,block,p); + _mi_page_profile_on_free(page,block,p); } - else + else #endif #if MI_GUARDED - if (block->next == MI_BLOCK_TAG_GUARDED) { - _mi_page_block_unguard(page, block, p); - *was_guarded = true; + if (block->next == MI_BLOCK_TAG_GUARDED) { + _mi_page_block_unguard(page, block, p); + *was_guarded = true; } else #endif @@ -162,7 +186,7 @@ static inline mi_block_t* mi_page_ptr_block_check(mi_page_t* page, void* p, bool static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, void* p) mi_attr_noexcept { mi_assert_internal(p!=NULL && page != NULL); bool was_guarded = false; - mi_block_t* block = mi_page_ptr_block_check(page,p,&was_guarded); + mi_block_t* block = mi_page_ptr_block_check(page,p,&was_guarded); // mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p)); // mi_block_check_profiled(page,block,p); // const bool was_guarded = mi_block_check_unguard(page, block, p); @@ -193,16 +217,16 @@ static mi_decl_forceinline bool mi_ptr_page_is_valid_ex(const void* p, const cha { MI_UNUSED_RELEASE(msg); MI_UNUSED(free_small); #if MI_DEBUG - if mi_unlikely(!_mi_is_aligned(p,MI_SIZE_SIZE) && !mi_option_is_enabled(mi_option_guarded_precise)) { + if mi_unlikely(!mi_is_aligned(p,MI_SIZE_SIZE) && !mi_option_is_enabled(mi_option_unsafe_guarded_precise)) { _mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p); return false; } #endif - + mi_page_t* page; - #if MI_PAGE_META_SMALL_IS_ALIGNED - if (free_small) { - const uintptr_t up = _mi_align_down((uintptr_t)p,MI_SMALL_PAGE_SIZE); // like this for codegen on gcc + #if MI_PAGE_META_SMALL_IS_ALIGNED + if (free_small) { + const uintptr_t up = mi_align_down((uintptr_t)p,MI_SMALL_PAGE_SIZE); // like this for codegen on gcc if mi_unlikely(up==0 && check_p_for_null) { return false; } page = (mi_page_t*)up; } @@ -210,10 +234,10 @@ static mi_decl_forceinline bool mi_ptr_page_is_valid_ex(const void* p, const cha #endif { #if MI_PAGE_META_IS_ALIGNED - { page = _mi_aligned_ptr_page0(p); } + { page = mi_aligned_ptr_page0(p); } #else - { page = _mi_ptr_page(p); } - #endif + { page = mi_ptr_page(p); } + #endif if mi_unlikely(page==NULL && check_p_for_null) { #if MI_DEBUG if (p!=NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } @@ -222,24 +246,32 @@ static mi_decl_forceinline bool mi_ptr_page_is_valid_ex(const void* p, const cha } } #if MI_DEBUG - mi_page_t* const cpage = _mi_checked_ptr_page(p); + mi_page_t* const cpage = mi_checked_ptr_page(p); if mi_unlikely(cpage==NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } #endif - - #if MI_PAGE_META_IS_ALIGNED + + #if MI_PAGE_META_IS_ALIGNED #if MI_PAGE_META_SMALL_IS_ALIGNED - if (free_small) { mi_assert_internal(NULL == mi_atomic_load_ptr_acquire(mi_page_t,&page->self)); } + if (free_small) { mi_assert_internal(NULL == mi_atomic_load_ptr_acquire(mi_page_t,&page->self)); } else #elif MI_SMALL_PAGE_SIZE == MI_ARENA_SLICE_SIZE && !MI_GUARDED // for mi_free_small we can avoid a load-acquire (but not when guarded as that may still allocate large blocks) if (free_small) { mi_assert_internal(page == mi_atomic_load_ptr_acquire(mi_page_t,&page->self)); } else #endif - { page = mi_atomic_load_ptr_acquire(mi_page_t,&page->self); } + { + // The `self` load can be relaxed here as we free a known pointer and thus it has been synchronized. + // However, the ThreadSanitizer seems to require an acquire load to properly track memory dependencies. + #if MI_TSAN + page = mi_atomic_load_ptr_acquire(mi_page_t,&page->self); + #else + page = mi_atomic_load_ptr_relaxed(mi_page_t,&page->self); + #endif + } #endif mi_assert_internal(page!=NULL); - mi_assert(cpage==page /* page_map lookup should be the same as aligned lookup */ ); + mi_assert(cpage==page /* page_map lookup should be the same as aligned lookup */ ); #if !MI_GUARDED if (free_small) { mi_assert_internal(page->block_size <= MI_SMALL_MAX_OBJ_SIZE); /* free small should only be called on small pages */ } #endif @@ -258,7 +290,7 @@ static mi_decl_forceinline mi_page_t* mi_ptr_page_validate(const void* p, const // Free a block // Fast path written carefully to prevent register spilling on the stack -static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t* pblock_size, bool allow_reclaim) +static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t* pblock_size, bool allow_reclaim) { mi_assert_internal(p!=NULL && page!=NULL); if (pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); } @@ -286,16 +318,16 @@ static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t } } -void mi_free(void* p) mi_attr_noexcept { - mi_page_t* page; - if mi_likely(mi_ptr_page_is_valid(p,"mi_free",&page)) { +void mi_free(void* p) mi_attr_noexcept { + mi_page_t* page = NULL; + if mi_likely(mi_ptr_page_is_valid(p,"mi_free",&page)) { mi_free_nonnull(p, page, NULL, true /* allow reclaim? */); } } void mi_ufree(void* p, size_t* pblock_size) mi_attr_noexcept { - mi_page_t* page; - if mi_likely(mi_ptr_page_is_valid(p,"mi_ufree",&page)) { + mi_page_t* page = NULL; + if mi_likely(mi_ptr_page_is_valid(p,"mi_ufree",&page)) { mi_free_nonnull(p, page, pblock_size, true /* allow reclaim? */); } else { @@ -303,30 +335,59 @@ void mi_ufree(void* p, size_t* pblock_size) mi_attr_noexcept { } } +// Free a pointer that is potentially allocated in a different sub-process +void _mi_free_subproc_safe(void* p) mi_attr_noexcept { + mi_page_t* page = NULL; + if mi_likely(mi_ptr_page_is_valid(p,"_mi_free_subproc_safe",&page)) { + // const bool allow_reclaim = (_mi_subproc() == mi_page_subproc(page)); + mi_free_nonnull(p, page, NULL, false /* allow reclaim */); + } +} + void mi_free_small(void* p) mi_attr_noexcept { - mi_page_t* page; - if mi_likely(mi_ptr_page_is_valid_ex(p,"mi_free_small",true /* is_small? */,true /*check p for null*/, &page)) { + mi_page_t* page = NULL; + if mi_likely(mi_ptr_page_is_valid_ex(p,"mi_free_small",true /* is_small? */,true /*check p for null*/, &page)) { mi_free_nonnull(p, page, NULL, true /* allow reclaim? */); } } void mi_free_small_nonnull(void* p) mi_attr_noexcept { mi_assert(p!=NULL); - mi_page_t* page; - if mi_likely(mi_ptr_page_is_valid_ex(p,"mi_free_small_nonnull",true /* is_small? */,false /*check p for null*/, &page)) { + mi_page_t* page = NULL; + if mi_likely(mi_ptr_page_is_valid_ex(p,"mi_free_small_nonnull",true /* is_small? */,false /*check p for null*/, &page)) { mi_free_nonnull(p, page, NULL, true /* allow reclaim? */); } } -// Free a pointer that is potentially allocated in a different sub-process -void _mi_free_subproc_safe(void* p) mi_attr_noexcept { - mi_page_t* page; - if mi_likely(mi_ptr_page_is_valid(p,"_mi_free_subproc_safe",&page)) { - // const bool allow_reclaim = (_mi_subproc() == mi_page_subproc(page)); - mi_free_nonnull(p, page, NULL, false /* allow reclaim */); +// For runtime systems: Free a pointer that is guaranteed to be small, and in a page owned by the current thread. +static mi_decl_forceinline void mi_free_small_local_ex(void* p, bool check_p_for_null) mi_attr_noexcept { + mi_assert(p!=NULL); + mi_page_t* page = NULL; + if mi_likely(mi_ptr_page_is_valid_ex(p,"mi_free_small_local_ex", true /* is_small? */, check_p_for_null /*check p for null*/, &page)) { + mi_assert_internal(mi_page_thread_id(page) == _mi_thread_id()); + if mi_likely(mi_page_flags(page) == 0) { + // thread-local, aligned, and not a full page + mi_block_t* const block = mi_validate_block_from_ptr(page,p); + mi_free_block_local(page, block, false /* was guarded */, false /* no need to check if the page is full */); + return; + } + else { + // page is local, but is full or contains (inner) aligned blocks; use generic path + mi_free_generic_local(page, p); + } } } +void mi_free_small_local(void* p) mi_attr_noexcept { + mi_free_small_local_ex(p, true /* check null */); +} + +void mi_free_small_local_nonnull(void* p) mi_attr_noexcept { + mi_free_small_local_ex(p, false /* check null */); +} + + + // ------------------------------------------------------ // Free variants // ------------------------------------------------------ @@ -337,31 +398,30 @@ void mi_free_size(void* p, size_t size) mi_attr_noexcept { const mi_page_t* const page = mi_ptr_page_validate(p,"mi_free_size"); if (page==NULL) return; mi_assert(p!=NULL); - const mi_block_t* block = _mi_page_ptr_unalign(page, p); + const mi_block_t* block = _mi_page_ptr_unalign(page, p); const size_t usable = _mi_page_usable_size(page,p); - if mi_unlikely(size > usable) { - const bool is_guarded = mi_block_ptr_is_guarded(block,p); + const bool is_guarded = mi_block_ptr_is_guarded(block,p); // guarded blocks are always in a non-small page + if mi_unlikely(size > usable) { if (!is_guarded) { - _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the size %zu is greater than the usable size %zu\n", p, size, usable); + _mi_error_message(EINVAL, "pointer %p is freed with 'mi_free_size' but the size %zu is greater than the usable size %zu\n", p, size, usable); mi_free(p); return; } } - // const size_t is_aligned = ((void*)block != p); - // if mi_unlikely(size <= MI_SMALL_SIZE_MAX && mi_page_block_size(page) > mi_good_size((is_aligned ? 2 : 1)*MI_SMALL_SIZE_MAX)) { // note: we check *2 in case it was over-aligned - // const bool is_guarded = mi_block_ptr_is_guarded(block,p); - // if (!is_guarded) { - // _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the given size %zu is less than the allocated block size %zu\n (maybe a `new[]` was matched with `delete` instead of `delete[]`?)\n", p, size, mi_page_block_size(page)); - // mi_free(p); - // return; - // } - // } + else if mi_unlikely(size <= MI_SMALL_SIZE_MAX && !mi_page_is_small(page) && !is_guarded) { + _mi_error_message(EINVAL, "pointer %p is freed with 'mi_free_size' with size %zu (<= MI_SMALL_SIZE_MAX), but it was allocated in a non-small page with block size %zu -- maybe a mismatched delete operator?\n", p, size, mi_page_block_size(page)); + mi_free(p); + return; + } #endif - #if MI_PAGE_META_SMALL_IS_ALIGNED || MI_PAGE_META_IS_ALIGNED + + // note: only use `mi_free_small` if small allocations are aligned. + // this allows building with `-DMI_OPT_FREE_SMALL=OFF` in case wrong sizes are passed to `mi_free_size`, see issue #1423 + #if MI_PAGE_META_SMALL_IS_ALIGNED if mi_likely(size <= MI_SMALL_SIZE_MAX) { - mi_free_small(p); + mi_free_small(p); } - else + else #endif { mi_free(p); @@ -389,7 +449,7 @@ void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { // checked free bool mi_cfree(void* p) mi_attr_noexcept { - mi_page_t* const page = _mi_checked_ptr_page(p); + mi_page_t* const page = mi_checked_ptr_page(p); if mi_likely(page!=NULL) { mi_free_nonnull(p, page, NULL, true /* allow reclaim? */); return true; @@ -407,13 +467,30 @@ bool mi_cfree(void* p) mi_attr_noexcept { // 3. if it went to < 7/8th used, re-abandon to be mapped (so it can be found by theaps looking for free pages) // -------------------------------------------------------------------------------------------- -// Helper for mi_free_try_collect_mt: free if the page has no more used blocks (this is updated by `_mi_page_free_collect(_partly)`) -static bool mi_abandoned_page_try_free(mi_page_t* page) +// Consume an empty page, reclaiming delayed pages if this thread has a theap for their heap. +static bool mi_abandoned_page_collect_empty(mi_page_t* page) { if (!mi_page_all_free(page)) return false; + const bool should_delay = _mi_page_qsbr_should_delay(page); // first remove it from the abandoned pages in the arena (if mapped, this might wait for any readers to finish) _mi_arenas_page_unabandon(page,NULL); - _mi_arenas_page_free(page,NULL); // we can now free the page directly + if mi_likely(!should_delay) { + _mi_page_qsbr_clear(page); + _mi_arenas_page_free(page,NULL); + } + else { + mi_heap_t* const heap = mi_page_heap(page); + mi_theap_t* const theap = mi_heap_theap(heap); + if (theap != NULL) { + // reclaim such that it gets part of the theap's qsbr list. + _mi_theap_page_reclaim(theap, page); + } + else { + mi_assert_internal(false); + // abandon as empty; it can only be collected by _mi_heap_collect_abandoned at this point + _mi_arenas_page_abandon(page,NULL); + } + } return true; } @@ -422,7 +499,7 @@ static bool mi_abandoned_page_try_reabandon_to_mapped(mi_page_t* page) { // if the page is unmapped, try to reabandon so it can possibly be mapped and found for allocations // We only reabandon if a full page starts to have enough blocks available to prevent immediate re-abandon of a full page - if (mi_page_is_mostly_used(page)) return false; // not too full + if (mi_page_is_mostly_used(page)) return false; // not too full (1/8th should be free) if (page->memid.memkind != MI_MEM_ARENA || mi_page_is_abandoned_mapped(page)) return false; // and not already mapped (or unmappable) mi_assert(!mi_page_is_full(page)); @@ -437,7 +514,7 @@ static void mi_abandoned_page_unown_from_free(mi_page_t* page, mi_block_t* page_ mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(!mi_page_all_free(page)); // try to cas atomically the original free list (`mt_free`) back with the ownership cleared. - mi_thread_free_t tf_expect = mi_tf_create(page_thread_free,true); + mi_thread_free_t tf_expect = mi_tf_create(page_thread_free,true); mi_thread_free_t tf_new = mi_tf_set_owned(tf_expect,false); while mi_unlikely(!mi_atomic_cas_strong_acq_rel(&page->xthread_free, &tf_expect, tf_new)) { mi_assert_internal(mi_tf_is_owned(tf_expect)); @@ -445,13 +522,13 @@ static void mi_abandoned_page_unown_from_free(mi_page_t* page, mi_block_t* page_ while (mi_tf_block(tf_expect) != NULL) { // if there were concurrent updates to the thread-free list, we retry to free or reabandon to mapped (if it became !mosty_used). _mi_page_free_collect(page,false); // update used count - if (mi_abandoned_page_try_free(page)) return; + if (mi_abandoned_page_collect_empty(page)) return; if (mi_abandoned_page_try_reabandon_to_mapped(page)) return; // otherwise continue un-owning tf_expect = mi_atomic_load_acquire(&page->xthread_free); } // and try again to release ownership - mi_subproc_stat_counter_increase(mi_page_subproc(page), pages_unabandon_busy_wait, 1); + mi_subproc_stat_counter_increase(mi_page_subproc(page), pages_unown_cas_retry, 1); mi_assert_internal(mi_tf_block(tf_expect)==NULL); tf_new = mi_tf_create(NULL, false); } @@ -476,17 +553,17 @@ static mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, long mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(!mi_page_all_free(page)); - mi_assert_internal(page->block_size <= MI_MEDIUM_MAX_OBJ_SIZE); + mi_assert_internal(page->block_size <= MI_MAX_RECLAIM_OBJ_SIZE); mi_assert_internal(reclaim_on_free >= 0); - + // dont reclaim if we just have terminated this thread and we should // not reinitialize the theap for this thread. (can happen due to thread-local destructors for example -- issue #944) if (!_mi_thread_is_initialized()) return false; - // get our theap + // get our theap mi_theap_t* const theap = _mi_page_associated_theap_peek(page); if (theap==NULL || theap->tld==NULL || !theap->allow_page_reclaim) return false; // see issue #1289 - + // todo: cache `is_in_threadpool` and `exclusive_arena` directly in the theap for performance? // set max_reclaim limit long max_reclaim = 0; @@ -495,12 +572,15 @@ static mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, long max_reclaim = _mi_option_get_fast(theap->tld->is_in_threadpool ? mi_option_page_cross_thread_max_reclaim : mi_option_page_max_reclaim); } else if (reclaim_on_free == 1 && // if cross-thread is allowed - !theap->tld->is_in_threadpool && // and we are not part of a threadpool - !mi_page_is_mostly_used(page) && // and the page is not too full - _mi_arena_memid_is_suitable(page->memid, _mi_theap_heap(theap)->exclusive_arena)) { // and it fits our memory + !theap->tld->is_in_threadpool && // and we are not part of a threadpool + !mi_page_is_used_at_frac(page,4) && // and at least 1/4th of the page is free + _mi_arena_memid_is_suitable(page->memid, mi_theap_heap(theap)->exclusive_arena)) { // and it fits our memory // across threads max_reclaim = _mi_option_get_fast(mi_option_page_cross_thread_max_reclaim); } + else { + return false; + } // are we within the reclaim limit? if (max_reclaim >= 0 && !mi_page_queue_len_is_atmost(theap, page->block_size, max_reclaim)) { @@ -523,40 +603,40 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(mt_free != NULL); // mi_assert_internal(!allow_reclaim || _mi_subproc() == mi_page_subproc(page)); // never reclaim across subprocesses - + // we own the page now, and it is safe to collect the thread atomic free list if (page->block_size <= MI_SMALL_SIZE_MAX) { // use the `_partly` version to avoid atomic operations since we already have the `mt_free` pointing into the thread free list // (after this the `used` count might be too high (as some blocks may have been concurrently added to the thread free list and are yet uncounted). // however, if the page became completely free, the used count is guaranteed to be 0.) mi_assert_internal(page->reserved>=16); // below this even one freed block goes from full to no longer mostly used. - mt_free = _mi_page_free_collect_partly(page, mt_free); + mt_free = _mi_page_free_collect_partly(page, mt_free); } else { - // for larger blocks we use the regular collect + // for larger blocks we use the regular collect _mi_page_free_collect(page,false /* no force */); mt_free = NULL; // expected page->xthread_free value after collection } - const long reclaim_on_free = allow_reclaim && _mi_option_get_fast(mi_option_page_reclaim_on_free); + const long reclaim_on_free = mi_heap_reclaim_on_free(mi_page_heap(page)); #if MI_DEBUG > 1 if (mi_page_is_singleton(page)) { mi_assert_internal(mi_page_all_free(page)); } if (mi_page_is_full(page)) { mi_assert(mi_page_is_mostly_used(page)); } #endif // try to: 1. free it, 2. reclaim it, or 3. reabandon it to be mapped - if (mi_abandoned_page_try_free(page)) return; - if (page->block_size <= MI_MEDIUM_MAX_OBJ_SIZE && reclaim_on_free >= 0) { // early test for better codegen + if (mi_abandoned_page_collect_empty(page)) return; + if (allow_reclaim && page->block_size <= MI_MAX_RECLAIM_OBJ_SIZE && reclaim_on_free >= 0) { // early test for better codegen if (mi_abandoned_page_try_reclaim(page, reclaim_on_free)) return; } if (mi_abandoned_page_try_reabandon_to_mapped(page)) return; - + // otherwise unown the page again mi_abandoned_page_unown_from_free(page, mt_free); } // ------------------------------------------------------ -// Usable size +// Usable size // ------------------------------------------------------ // Bytes available in a block @@ -589,6 +669,13 @@ mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept { return _mi_page_usable_size(page,p); } +mi_decl_nodiscard size_t mi_thread_id_of(const void* p) mi_attr_noexcept { + if (p == NULL) return 0; + const mi_page_t* const page = mi_ptr_page(p); + const mi_threadid_t tid = mi_page_thread_id(page); + return (tid <= MI_THREADID_DETACHED ? 0 : tid); +} + // ------------------------------------------------------ // Deprecated: double free is usually checked with padding now @@ -597,7 +684,7 @@ mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept { // This is somewhat expensive so only enabled for secure mode 4 // ------------------------------------------------------ -#if MI_SECURE>=3 && !MI_PADDING +#if MI_SECURE>=3 && !MI_PADDING // linear check if the free list contains a specific element static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem, const char* list_kind) { const size_t max_count = page->capacity; // can never hold more blocks than the capacity @@ -605,7 +692,7 @@ static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, cons while (list != NULL && count <= max_count) { // double-free can create cycles so we limit the number of iterations if (elem==list) return true; list = mi_block_next(page, list); - count++; + count++; } if mi_unlikely(count > max_count) { _mi_error_message(EFAULT, "corrupted %s list (possibly due to a double free)\n", list_kind); @@ -629,8 +716,8 @@ static mi_decl_noinline bool mi_check_double_freex(const mi_page_t* page, const // Used for double free checking to avoid checking free lists too frequently static inline bool mi_block_could_be_double_free(const mi_page_t* page, const mi_block_t* block) { mi_block_t* n = mi_block_nextx(page,block,page->keys); - return (_mi_is_aligned(block,MI_SIZE_SIZE) && // quick check: aligned pointer? - (n==NULL || mi_page_contains_address(page,n))); // quick check: in the same page or NULL? + return (mi_is_aligned(block,MI_SIZE_SIZE) && // quick check: aligned pointer? + (n==NULL || mi_page_contains_address(page,n))); // quick check: in the same page or NULL? } // check if `block` was free'd before @@ -768,7 +855,7 @@ mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, co else { _mi_error_message(EFAULT, "buffer overflow in heap block %p of size %zu: write after %zu bytes\n", block, *usable_size, wrong ); } - return false; + return false; } return true; } @@ -786,14 +873,14 @@ mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, co // only maintain stats for smaller objects if requested #if 0 && (MI_STATS>0) static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { - MI_UNUSED(block); + MI_UNUSED(block); mi_theap_t* theap = _mi_theap_default(); mi_lock_t* lock = NULL; mi_subproc_t* const subproc = mi_page_subproc(page); mi_theap_t* const theap_meta = subproc->theap_meta; if mi_unlikely(!mi_theap_is_initialized(theap) || // can happen if free'd after thread_done was called (usually a thread cleanup call by the OS) // page->theap == subproc->theap_meta .. but we cannot read `theap` if we don't own the page - (theap_meta != NULL && mi_page_thread_id(page) == theap_meta->tld->thread_id)) { + (theap_meta != NULL && mi_page_thread_id(page) == theap_meta->tld->thread_id)) { theap = theap_meta; lock = &subproc->theap_meta_lock; mi_lock_acquire(lock); diff --git a/system/lib/mimalloc/src/page-queue.c b/system/lib/mimalloc/src/alloc/page-queue.c similarity index 84% rename from system/lib/mimalloc/src/page-queue.c rename to system/lib/mimalloc/src/alloc/page-queue.c index c8e9b5425ce5a..a9d8c42e3007b 100644 --- a/system/lib/mimalloc/src/page-queue.c +++ b/system/lib/mimalloc/src/alloc/page-queue.c @@ -1,5 +1,5 @@ /*---------------------------------------------------------------------------- -Copyright (c) 2018-2024, Microsoft Research, Daan Leijen +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -9,13 +9,9 @@ terms of the MIT license. A copy of the license can be found in the file Definition of page queues for each block size ----------------------------------------------------------- */ -#ifndef MI_IN_PAGE_C -#error "this file should be included from 'page.c'" -// include to help an IDE #include "mimalloc.h" -#include "mimalloc/internal.h" -#include "mimalloc/atomic.h" -#endif +#include "mimalloc/types.h" +#include "page-queue.h" /* ----------------------------------------------------------- Minimal alignment in machine words (i.e. `sizeof(void*)`) @@ -32,27 +28,6 @@ terms of the MIT license. A copy of the license can be found in the file #endif -/* ----------------------------------------------------------- - Queue query ------------------------------------------------------------ */ - - -static inline bool mi_page_queue_is_huge(const mi_page_queue_t* pq) { - return (pq->block_size == (MI_LARGE_MAX_OBJ_SIZE+sizeof(uintptr_t))); -} - -static inline bool mi_page_queue_is_full(const mi_page_queue_t* pq) { - return (pq->block_size == (MI_LARGE_MAX_OBJ_SIZE+(2*sizeof(uintptr_t)))); -} - -static inline bool mi_page_queue_is_special(const mi_page_queue_t* pq) { - return (pq->block_size > MI_LARGE_MAX_OBJ_SIZE); -} - -static inline size_t mi_page_queue_count(const mi_page_queue_t* pq) { - return pq->count; -} - /* ----------------------------------------------------------- Bins ----------------------------------------------------------- */ @@ -116,7 +91,7 @@ mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexce return _mi_bin_size(mi_bin(size + MI_PADDING_SIZE)); } else if (size <= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { - return _mi_align_up(size + MI_PADDING_SIZE,_mi_os_page_size()); + return mi_align_up(size + MI_PADDING_SIZE,_mi_os_page_size()); } else { return size; @@ -124,7 +99,7 @@ mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexce } #if (MI_DEBUG>1) -static bool mi_page_queue_contains(mi_page_queue_t* queue, const mi_page_t* page) { +bool mi_page_queue_contains(mi_page_queue_t* queue, const mi_page_t* page) { mi_assert_internal(page != NULL); mi_page_t* list = queue->first; while (list != NULL) { @@ -138,12 +113,6 @@ static bool mi_page_queue_contains(mi_page_queue_t* queue, const mi_page_t* page #endif -#if (MI_DEBUG>1) -static bool mi_theap_contains_queue(const mi_theap_t* theap, const mi_page_queue_t* pq) { - return (pq >= &theap->pages[0] && pq <= &theap->pages[MI_BIN_FULL]); -} -#endif - bool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq) { MI_UNUSED_RELEASE(theap); if (pq==NULL) return false; @@ -184,7 +153,7 @@ size_t _mi_page_stats_bin(const mi_page_t* page) { return bin; } -static mi_page_queue_t* mi_theap_page_queue_of(mi_theap_t* theap, const mi_page_t* page) { +mi_page_queue_t* mi_theap_page_queue_of(mi_theap_t* theap, const mi_page_t* page) { mi_assert_internal(theap!=NULL); const size_t bin = mi_page_bin(page); mi_page_queue_t* pq = &theap->pages[bin]; @@ -194,13 +163,6 @@ static mi_page_queue_t* mi_theap_page_queue_of(mi_theap_t* theap, const mi_page_ return pq; } -static mi_page_queue_t* mi_page_queue_of(const mi_page_t* page) { - mi_theap_t* theap = mi_page_theap(page); - mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page); - mi_assert_expensive(mi_page_queue_contains(pq, page)); - return pq; -} - // The current small page array is for efficiency and for each // small size (up to 256) it points directly to the page for that // size without having to compute the bin. This means when the @@ -226,7 +188,7 @@ static inline void mi_theap_queue_first_update(mi_theap_t* theap, const mi_page_ } else { // find previous size; due to minimal alignment upto 3 previous bins may need to be skipped - mi_assert_internal(pq > &theap->pages[0]); // since idx > 1 + mi_assert_internal(pq > &theap->pages[0]); // since idx > 1 size_t bin = mi_bin(size); const mi_page_queue_t* prev = pq - 1; while( bin == mi_bin(prev->block_size) && prev > &theap->pages[0]) { @@ -249,7 +211,7 @@ static bool mi_page_queue_is_empty(mi_page_queue_t* queue) { } */ -static void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) { +void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) { mi_assert_internal(page != NULL); mi_assert_expensive(mi_page_queue_contains(queue, page)); mi_assert_internal(queue->count >= 1); @@ -267,6 +229,10 @@ static void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) { mi_theap_queue_first_update(theap,queue); } theap->page_count--; + if (page->memid.memkind != MI_MEM_ARENA) { + mi_assert_internal(theap->page_count_os > 0); + theap->page_count_os--; + } queue->count--; page->next = NULL; page->prev = NULL; @@ -274,7 +240,7 @@ static void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) { } -static void mi_page_queue_push(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) { +void mi_page_queue_push(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) { mi_assert_internal(mi_page_theap(page) == theap); mi_assert_internal(!mi_page_queue_contains(queue, page)); #if MI_HUGE_PAGE_ABANDON @@ -301,9 +267,10 @@ static void mi_page_queue_push(mi_theap_t* theap, mi_page_queue_t* queue, mi_pag // update direct mi_theap_queue_first_update(theap, queue); theap->page_count++; + if (page->memid.memkind != MI_MEM_ARENA) { theap->page_count_os++; } } -static void mi_page_queue_push_at_end(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) { +void mi_page_queue_push_at_end(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) { mi_assert_internal(mi_page_theap(page) == theap); mi_assert_internal(!mi_page_queue_contains(queue, page)); @@ -330,24 +297,7 @@ static void mi_page_queue_push_at_end(mi_theap_t* theap, mi_page_queue_t* queue, mi_theap_queue_first_update(theap, queue); } theap->page_count++; -} - -static void mi_page_queue_move_to_front(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) { - mi_assert_internal(mi_page_theap(page) == theap); - mi_assert_internal(mi_page_queue_contains(queue, page)); - if (queue->first == page) return; - mi_page_queue_remove(queue, page); - mi_page_queue_push(theap, queue, page); - mi_assert_internal(queue->first == page); -} - -static void mi_page_queue_move_to_back(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) { - mi_assert_internal(mi_page_theap(page) == theap); - mi_assert_internal(mi_page_queue_contains(queue, page)); - if (queue->last == page) return; - mi_page_queue_remove(queue, page); - mi_page_queue_push_at_end(theap, queue, page); - mi_assert_internal(queue->last == page); + if (page->memid.memkind != MI_MEM_ARENA) { theap->page_count_os++; } } static void mi_page_queue_enqueue_from_ex(mi_page_queue_t* to, mi_page_queue_t* from, bool enqueue_at_end, mi_page_t* page) { @@ -422,11 +372,29 @@ static void mi_page_queue_enqueue_from_ex(mi_page_queue_t* to, mi_page_queue_t* mi_page_set_in_full(page, mi_page_queue_is_full(to)); } -static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) { +void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) { mi_page_queue_enqueue_from_ex(to, from, true /* enqueue at the end */, page); } -static void mi_page_queue_enqueue_from_full(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) { +void mi_page_queue_enqueue_from_full(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) { // note: we could insert at the front to increase reuse, but it slows down certain benchmarks (like `alloc-test`) mi_page_queue_enqueue_from_ex(to, from, true /* enqueue at the end of the `to` queue? */, page); } + +void mi_page_queue_move_to_front(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) { + mi_assert_internal(mi_page_theap(page) == theap); + mi_assert_internal(mi_page_queue_contains(queue, page)); + if (queue->first == page) return; + mi_page_queue_remove(queue, page); + mi_page_queue_push(theap, queue, page); + mi_assert_internal(queue->first == page); +} + +void mi_page_queue_move_to_back(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) { + mi_assert_internal(mi_page_theap(page) == theap); + mi_assert_internal(mi_page_queue_contains(queue, page)); + if (queue->last == page) return; + mi_page_queue_remove(queue, page); + mi_page_queue_push_at_end(theap, queue, page); + mi_assert_internal(queue->last == page); +} diff --git a/system/lib/mimalloc/src/alloc/page-queue.h b/system/lib/mimalloc/src/alloc/page-queue.h new file mode 100644 index 0000000000000..3a7096d3b6853 --- /dev/null +++ b/system/lib/mimalloc/src/alloc/page-queue.h @@ -0,0 +1,57 @@ +/*---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#pragma once +#ifndef MI_PAGE_QUEUE_H +#define MI_PAGE_QUEUE_H + +#include "mimalloc.h" +#include "mimalloc/types.h" +#include "mimalloc/alloc/page.h" + +#if (MI_DEBUG>1) +bool mi_page_queue_contains(mi_page_queue_t* queue, const mi_page_t* page); + +static inline bool mi_theap_contains_queue(const mi_theap_t* theap, const mi_page_queue_t* pq) { + return (pq >= &theap->pages[0] && pq <= &theap->pages[MI_BIN_FULL]); +} +#endif + +mi_page_queue_t* mi_theap_page_queue_of(mi_theap_t* theap, const mi_page_t* page); +void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page); +void mi_page_queue_push(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page); +void mi_page_queue_push_at_end(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page); +void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page); +void mi_page_queue_enqueue_from_full(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page); +void mi_page_queue_move_to_front(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page); +void mi_page_queue_move_to_back(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page); + + +static inline bool mi_page_queue_is_huge(const mi_page_queue_t* pq) { + return (pq->block_size == (MI_LARGE_MAX_OBJ_SIZE+sizeof(uintptr_t))); +} + +static inline bool mi_page_queue_is_full(const mi_page_queue_t* pq) { + return (pq->block_size == (MI_LARGE_MAX_OBJ_SIZE+(2*sizeof(uintptr_t)))); +} + +static inline bool mi_page_queue_is_special(const mi_page_queue_t* pq) { + return (pq->block_size > MI_LARGE_MAX_OBJ_SIZE); +} + +static inline size_t mi_page_queue_count(const mi_page_queue_t* pq) { + return pq->count; +} + +static inline mi_page_queue_t* mi_page_queue_of(const mi_page_t* page) { + mi_theap_t* theap = mi_page_theap(page); + mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page); + mi_assert_expensive(mi_page_queue_contains(pq, page)); + return pq; +} + +#endif diff --git a/system/lib/mimalloc/src/page.c b/system/lib/mimalloc/src/alloc/page.c similarity index 54% rename from system/lib/mimalloc/src/page.c rename to system/lib/mimalloc/src/alloc/page.c index aa669cdd02928..741725707eab6 100644 --- a/system/lib/mimalloc/src/page.c +++ b/system/lib/mimalloc/src/alloc/page.c @@ -1,33 +1,23 @@ /*---------------------------------------------------------------------------- -Copyright (c) 2018-2024, Microsoft Research, Daan Leijen +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -/* ----------------------------------------------------------- - The core of the allocator. Every segment contains - pages of a certain block size. The main function - exported is `mi_malloc_generic`. ------------------------------------------------------------ */ - #include "mimalloc.h" -#include "mimalloc/internal.h" -#include "mimalloc/atomic.h" -#include "mimalloc/prim.h" -#include "mimalloc/prim-tls.h" - -/* ----------------------------------------------------------- - Definition of page queues for each block size ------------------------------------------------------------ */ - -#define MI_IN_PAGE_C -#include "page-queue.c" -#undef MI_IN_PAGE_C +#include "mimalloc/types.h" +#include "mimalloc/prim/prim-tls.h" +#include "mimalloc/util/random.h" +#include "mimalloc/util/stats.h" +#include "mimalloc/arena/arena-page.h" +#include "mimalloc/alloc/page.h" +#include "page-queue.h" +#include "page.h" /* ----------------------------------------------------------- - Page helpers + Page invariants ----------------------------------------------------------- */ // Index a block in a page @@ -38,16 +28,14 @@ static inline mi_block_t* mi_page_block_at(const mi_page_t* page, void* page_sta return (mi_block_t*)((uint8_t*)page_start + (i * block_size)); } -static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page); - #if (MI_DEBUG>=3) static size_t mi_page_list_count(mi_page_t* page, mi_block_t* head) { - mi_assert_internal(_mi_ptr_page(mi_page_start(page)) == page); + mi_assert_internal(mi_ptr_page(mi_page_start(page)) == page); const uint8_t* slice_start = mi_page_slice_start(page); - mi_assert_internal(_mi_is_aligned(slice_start,MI_PAGE_ALIGN)); + mi_assert_internal(mi_is_aligned(slice_start,MI_PAGE_ALIGN)); size_t count = 0; while (head != NULL) { - mi_assert_internal((uint8_t*)head - slice_start > (ptrdiff_t)MI_LARGE_PAGE_SIZE || page == _mi_ptr_page(head)); + mi_assert_internal((uint8_t*)head - slice_start > (ptrdiff_t)MI_LARGE_PAGE_SIZE || page == mi_ptr_page(head)); count++; head = mi_block_next(page, head); } @@ -85,10 +73,11 @@ static bool mi_page_is_valid_init(mi_page_t* page) { mi_assert_internal(mi_page_block_size(page) > 0); mi_assert_internal(mi_page_used(page) <= page->capacity); mi_assert_internal(page->capacity <= page->reserved); - + mi_assert_internal(page->heap!=NULL); mi_theap_t* const page_theap = _mi_heap_theap_peek(page->heap); - mi_assert_internal(page_theap == NULL || mi_page_theap(page)==page_theap || mi_page_theap(page)->tld->thread_id == MI_THREADID_DETACHED); + mi_assert_internal(page_theap == NULL || mi_page_theap(page)==page_theap || + mi_page_theap(page)->tld == NULL || mi_page_theap(page)->tld->thread_id == MI_THREADID_DETACHED); // const size_t bsize = mi_page_block_size(page); // uint8_t* start = mi_page_start(page); @@ -104,7 +93,7 @@ static bool mi_page_is_valid_init(mi_page_t* page) { mi_assert_expensive(mi_mem_is_zero(block + 1, ubsize - sizeof(mi_block_t))); } } - #endif + #endif #if !MI_TRACK_ENABLED && !MI_TSAN mi_block_t* tfree = mi_page_thread_free(page); @@ -131,7 +120,8 @@ bool _mi_page_is_valid(mi_page_t* page) { //mi_assert_internal(!_mi_process_is_initialized); mi_assert_internal(page->heap!=NULL); mi_theap_t* const page_theap = _mi_heap_theap_peek(page->heap); - mi_assert_internal(page_theap == NULL || mi_page_theap(page)==page_theap || mi_page_theap(page)->tld->thread_id == MI_THREADID_DETACHED); + mi_assert_internal(page_theap == NULL || mi_page_theap(page)==page_theap || + mi_page_theap(page)->tld == NULL || mi_page_theap(page)->tld->thread_id == MI_THREADID_DETACHED); { mi_page_queue_t* pq = mi_page_queue_of(page); mi_assert_internal(mi_page_queue_contains(pq, page)); @@ -145,27 +135,35 @@ bool _mi_page_is_valid(mi_page_t* page) { } #endif -#if MI_STATS || MI_SAMPLE +/* ----------------------------------------------------------- + Page statistics +----------------------------------------------------------- */ + + +#if MI_STATS || (MI_SAMPLE==1) // Gets the theap belonging to a page. static mi_theap_t* mi_theap_of_page(mi_page_t* page) { mi_theap_t* theap = page->theap; - if mi_unlikely(mi_page_thread_id(page) != _mi_prim_thread_id()) { + if mi_unlikely(mi_page_thread_id(page) != _mi_prim_thread_id()) { theap = _mi_page_associated_theap_peek(page); } - return theap; + return (theap != NULL && theap->tld != NULL ? theap : NULL); } #endif -#if MI_SAMPLE==1 /* for ==2, the countdown is already done at every `alloc.c:mi_page_alloc_zero` */ -static void mi_theap_adjust_sample_countdown(mi_theap_t* theap, mi_page_t* page, size_t alloc_count) +#if MI_SAMPLE==1 /* for ==2, the countdown is already done at every `alloc/alloc.c:mi_page_alloc_zero` */ + +// Adjust the theap->sample_countdown based on the current page allocation count +// in the `mi_malloc_generic` function this is countdown is checked +static void mi_theap_adjust_sample_countdown(mi_theap_t* theap, mi_page_t* page, size_t alloc_count) { - mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); mi_assert_internal(theap!=NULL); if (theap->sample_rate==0) return; const size_t last_alloc = mi_page_last_alloc(page); if (alloc_count <= last_alloc) return; - + // update countdown const size_t bsize = mi_page_usable_block_size(page); mi_assert_internal(bsize >= MI_PADDING_SIZE); const size_t alloc_diff = alloc_count - last_alloc; @@ -175,19 +173,19 @@ static void mi_theap_adjust_sample_countdown(mi_theap_t* theap, mi_page_t* page, } else { theap->sample_requested += (requested - theap->sample_countdown); - theap->sample_countdown = 0; + theap->sample_countdown = 0; } } #endif #if MI_STATS -// Merge stats from the page into the corresponding theap or heap. -static void mi_theap_page_merge_stats(mi_theap_t* theap, const mi_page_t* page, size_t alloc_count, size_t free_count ) { +// Merge stats from the page into the corresponding theap (or heap if theapx==NULL). +static void mi_theap_page_merge_stats(mi_theap_t* theapx, const mi_page_t* page, size_t alloc_count, size_t free_count ) { // get heap (as the theap might be NULL) mi_heap_t* const heap = mi_page_heap(page); - mi_assert_internal(theap == NULL || (theap->tld != NULL && _mi_thread_id() == theap->tld->thread_id)); - mi_theapx_stat_counter_increase(heap,theap,pages_stat_updates,1); - mi_theapx_stat_counter_increase(heap,theap,pages_stat_update_count, alloc_count + free_count); + mi_assert_internal(theapx == NULL || (theapx->tld != NULL && _mi_thread_id() == theapx->tld->thread_id)); + mi_theapx_stat_counter_increase(heap,theapx,pages_stat_updates,1); + mi_theapx_stat_counter_increase(heap,theapx,pages_stat_update_count, alloc_count + free_count); // allocation sizes const size_t bsize = mi_page_usable_block_size(page); @@ -198,60 +196,62 @@ static void mi_theap_page_merge_stats(mi_theap_t* theap, const mi_page_t* page, #endif mi_assert_internal(allocated <= INT64_MAX); // safe to cast to int64_t for stats mi_assert_internal(freed <= INT64_MAX); - + // adjust stats - if (bsize <= MI_LARGE_MAX_OBJ_SIZE) { - const size_t bin = _mi_bin(bsize); - // allocations - if (alloc_count > 0) { - mi_theapx_stat_counter_increase(heap, theap, malloc_normal_count, alloc_count); - mi_theapx_stat_increase(heap, theap, malloc_normal, allocated); - mi_theapx_stat_increase(heap, theap, malloc_bins[bin], alloc_count); - #if MI_STATS==1 - // use coarse total requested bytes - mi_theapx_stat_counter_increase(heap, theap, malloc_requested, requested); - #endif + const size_t bin = _mi_bin(bsize); + if (alloc_count > 0) { + mi_theapx_stat_counter_increase(heap, theapx, malloc_count, alloc_count); + mi_theapx_stat_increase(heap, theapx, allocated, allocated); + mi_theapx_stat_increase(heap, theapx, malloc_bins[bin], alloc_count); + if (bin==MI_BIN_HUGE) { mi_theapx_stat_increase(heap,theapx,malloc_huge,allocated); } // separate so we have a count and total + #if MI_STATS==1 + // use coarse total requested bytes + mi_theapx_stat_counter_increase(heap, theapx, malloc_requested, requested); + #endif + // deprecated + if (bin!=MI_BIN_HUGE) { + mi_theapx_stat_counter_increase(heap, theapx, malloc_normal_count, alloc_count); + mi_theapx_stat_increase(heap, theapx, malloc_normal, allocated); } - // frees - if (free_count > 0) { - mi_theapx_stat_decrease(heap, theap, malloc_normal, freed); - mi_theapx_stat_decrease(heap, theap, malloc_bins[bin], free_count); + else { + mi_theapx_stat_counter_increase(heap, theapx, malloc_huge_count, alloc_count); } } - else { - // allocations - mi_assert_internal(alloc_count<=1); - mi_assert_internal(free_count<=1); - if (alloc_count > 0) { - mi_theapx_stat_counter_increase(heap, theap, malloc_huge_count, alloc_count); - mi_theapx_stat_increase(heap, theap, malloc_huge, allocated); - #if MI_STATS==1 - // use coarse total requested bytes - mi_theapx_stat_counter_increase(heap, theap, malloc_requested, requested); - #endif - } - // frees - if (free_count > 0) { - mi_theapx_stat_decrease(heap, theap, malloc_huge, freed); + if (free_count > 0) { + mi_theapx_stat_counter_increase(heap, theapx, free_count, free_count); + mi_theapx_stat_counter_increase(heap, theapx, freed, freed); + mi_theapx_stat_decrease(heap, theapx, allocated, freed); + mi_theapx_stat_decrease(heap, theapx, malloc_bins[bin], free_count); + if (bin==MI_BIN_HUGE) { mi_theapx_stat_decrease(heap, theapx, malloc_huge, freed); } + // deprecated + if (bin!=MI_BIN_HUGE) { mi_theapx_stat_decrease(heap, theapx, malloc_normal, freed); } + } + + // merge the theap stats into the heap once N bytes were allocated or freed since the last merge + if (theapx != NULL) { + const size_t threshold = mi_option_get_size(mi_option_stats_merge_threshold); + const int64_t churn = theapx->stats.allocated.total + theapx->stats.freed.total; + if (threshold > 0 && churn >= (int64_t)threshold) { + _mi_theap_merge_stats(theapx); } } } -// Update stats for a page -static void mi_theap_page_update_stats(mi_theap_t* theap, mi_page_t* page) { +// Update stats for a page (and sample countdown). Can be called with a NULL `theapx` +static void mi_theap_page_update_stats(mi_theap_t* theapx, mi_page_t* page) { mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page)); - + // get stat counts const size_t used = mi_page_used(page); const size_t alloc_count = mi_page_alloc_count(page); - const size_t last_used = mi_page_last_used(page); + const size_t last_used = mi_page_last_used(page); mi_assert_internal(last_used + alloc_count >= used); mi_assert_internal(used <= UINT16_MAX); - const size_t free_count = last_used + alloc_count - used; + const size_t free_count = last_used + alloc_count - used; - #if MI_SAMPLE==1 // for ==2 it is already counted in every `alloc.c:mi_page_alloc_zero` - if (theap!=NULL) { mi_theap_adjust_sample_countdown(theap,page,alloc_count); } + #if MI_SAMPLE==1 // for ==2 it is already counted in every `alloc/alloc.c:mi_page_alloc_zero` + if (theapx!=NULL) { mi_theap_adjust_sample_countdown(theapx,page,alloc_count); } #endif // update stats? @@ -271,9 +271,10 @@ static void mi_theap_page_update_stats(mi_theap_t* theap, mi_page_t* page) { mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page)); - mi_theap_page_merge_stats(theap, page, alloc_count, free_count); + mi_theap_page_merge_stats(theapx, page, alloc_count, free_count); } +// Update stats for a page (and sample countdown) void _mi_page_update_stats(mi_page_t* page) { // called on abandoned pages etc. mi_theap_page_update_stats(mi_theap_of_page(page),page); } @@ -284,23 +285,27 @@ void _mi_page_update_stats(mi_page_t* page) { } #endif -static void mi_page_update_sample_countdown(mi_page_t* page) -{ - // adjust count down - const size_t alloc_count = mi_page_alloc_count(page); +// Update the sampling countdown based on the current allocation count of the page +void _mi_page_update_sample_countdown(mi_page_t* page) +{ + mi_assert_internal(mi_page_is_owned(page)); + // note: the threadid may not match and even be abandoned if we free multi-threaded (so we can not always use page->theap) + + // adjust count down + const size_t alloc_count = mi_page_alloc_count(page); if (alloc_count==0) { - return; + return; } - else if mi_unlikely(alloc_count>=0x8000) { // if the count could overflow, update stats so the counter is reset - _mi_page_update_stats(page); + else if mi_unlikely(alloc_count>=0x8000) { // if the count could overflow, update stats so the alloc_count is reset + _mi_page_update_stats(page); // calls mi_theap_adjust_sample_countdown if needed } - #if MI_SAMPLE==1 // if ==2 the countdown is already always counted in `alloc.c:mi_page_alloc_zero_ex` + #if MI_SAMPLE==1 // if ==2 the countdown is already always counted in `alloc/alloc.c:mi_page_alloc_zero_ex` else { - mi_theap_t* theap = mi_theap_of_page(page); - if (theap==NULL) return; + mi_theap_t* theap = mi_theap_of_page(page); // careful as the theap might not be valid + if (theap==NULL) return; // can be NULL on heap_destroy or metadata allocation etc. (and it is safe to ignore the sampling for now) mi_theap_adjust_sample_countdown(theap,page,alloc_count); - // update last_alloc to alloc_count - mi_assert_internal(alloc_count <= UINT16_MAX); + // update last_alloc to the current alloc_count + mi_assert_internal(alloc_count <= UINT16_MAX); #if MI_SIZE_SIZE >= 8 page->xused.used_alloc = (alloc_count << 48) | (page->xused.used_alloc & (~MI_ZU(0) >> 16)); #else @@ -308,7 +313,6 @@ static void mi_page_update_sample_countdown(mi_page_t* page) #endif mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page)); - } #endif } @@ -351,10 +355,12 @@ static void mi_page_thread_collect_to_local(mi_page_t* page, mi_block_t* head) mi_assert_internal(count <= UINT16_MAX); mi_assert_internal(mi_page_used(page) >= count); page->xused.used_alloc -= count; // page->used = page->used - (uint16_t)count; + if (mi_page_all_free(page)) { _mi_page_qsbr_clear(page); } } // Collect the local `thread_free` list using an atomic exchange. -static void mi_page_thread_free_collect(mi_page_t* page) +// Returns `true` if a non-empty thread free list was collected. +static bool mi_page_thread_free_collect(mi_page_t* page) { // atomically capture the thread free list mi_block_t* head; @@ -362,33 +368,22 @@ static void mi_page_thread_free_collect(mi_page_t* page) mi_thread_free_t tfree = mi_atomic_load_relaxed(&page->xthread_free); do { head = mi_tf_block(tfree); - if mi_likely(head == NULL) return; // return if the list is empty + if mi_likely(head == NULL) return false; // return if the list is empty tfreex = mi_tf_create(NULL,mi_tf_is_owned(tfree)); // set the thread free list to NULL } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tfree, tfreex)); // release is enough? mi_assert_internal(head != NULL); // and move it to the local list mi_page_thread_collect_to_local(page, head); -} - - -// returns `true` if after collection `mi_page_immediate_available` is true. -static inline bool mi_page_free_quick_collect(mi_page_t* page) { - if mi_likely(page->free != NULL) return true; - if (page->local_free == NULL) return false; - // move local_free to free - page->free = page->local_free; - page->local_free = NULL; - page->free_is_zero = false; - mi_page_update_sample_countdown(page); return true; } -void _mi_page_free_collect(mi_page_t* page, bool force) { + +bool _mi_page_free_collect(mi_page_t* page, bool force) { mi_assert_internal(page!=NULL); // collect the thread free list - mi_page_thread_free_collect(page); + const bool collected_xfree = mi_page_thread_free_collect(page); // and the local free list if (page->local_free != NULL) { @@ -410,19 +405,21 @@ void _mi_page_free_collect(mi_page_t* page, bool force) { page->local_free = NULL; page->free_is_zero = false; } - mi_page_update_sample_countdown(page); - } + _mi_page_update_sample_countdown(page); + } mi_assert_internal(!force || page->local_free == NULL); + return collected_xfree; } // Collect elements in the thread-free list starting at `head`. This is an optimized -// version of `_mi_page_free_collect` to be used from `free.c:_mi_free_collect_mt` that avoids atomic access to `xthread_free`. +// version of `_mi_page_free_collect` to be used from `alloc/free.c:_mi_free_collect_mt` that avoids atomic access to `xthread_free`. // returns a possibly updated expected value for the thread_free pointer. // // `head` must be in the `xthread_free` list. It will not collect `head` itself // so the `used` count is not fully updated in general. However, if the `head` is // the last remaining element, it will be collected and the used count will become `0` (so `mi_page_all_free` becomes true). mi_block_t* _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) { + mi_assert_internal(mi_page_is_owned(page)); if (head == NULL) return NULL; mi_block_t* next = mi_block_next(page,head); // we cannot collect the head element itself as `page->thread_free` may point to it (and we want to avoid atomic ops) if (next != NULL) { @@ -432,8 +429,8 @@ mi_block_t* _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) { page->free = page->local_free; page->local_free = NULL; page->free_is_zero = false; - mi_page_update_sample_countdown(page); - } + _mi_page_update_sample_countdown(page); + } } if (mi_page_used(page) == 1) { // all elements are free'd since we skipped the `head` element itself @@ -453,54 +450,58 @@ mi_block_t* _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) { ----------------------------------------------------------- */ // called from `mi_free` on a reclaim, and fresh_alloc if we get an abandoned page -void _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page) +bool _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page) { - mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); - mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(mi_ptr_page(mi_page_start(page))==page); mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(mi_page_is_abandoned(page)); mi_page_set_theap(page,theap); _mi_page_free_collect(page, false); // ensure used count is up to date - + mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page); mi_page_queue_push_at_end(theap, pq, page); + if (!mi_page_all_free(page)) { _mi_page_qsbr_clear(page); } mi_assert_expensive(_mi_page_is_valid(page)); + return (mi_page_all_free(page) && _mi_page_free(page, pq)); } void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) { _mi_page_free_collect(page, false); // ensure used count is up to date - if (mi_page_all_free(page)) { - _mi_page_free(page, pq); - } - else { - mi_page_queue_remove(pq, page); - mi_theap_t* theap = page->theap; - mi_page_set_theap(page, NULL); - page->theap = theap; // don't actually set theap to NULL so we can reclaim_on_free within the same theap - _mi_arenas_page_abandon(page, theap); - // _mi_arenas_collect(false, false, theap->tld); // allow purging - } + pq = mi_page_queue_of(page); // a preceding release attempt may have unfulled the page + if (mi_page_all_free(page) && _mi_page_free(page, pq)) return; + pq = mi_page_queue_of(page); // _mi_page_free may have moved an empty page out of the full queue + mi_page_queue_remove(pq, page); + mi_theap_t* theap = page->theap; + mi_page_qsbr_unlink(page); + mi_page_set_theap(page, NULL); + page->theap = theap; // don't actually set theap to NULL so we can reclaim_on_free within the same theap + _mi_arenas_page_abandon(page, theap); } // allocate a fresh page from an arena -static mi_page_t* mi_page_fresh_alloc(mi_theap_t* theap, mi_page_queue_t* pq, size_t block_size, size_t page_alignment) { +mi_page_t* _mi_page_fresh_alloc(mi_theap_t* theap, mi_page_queue_t* pq, size_t block_size, size_t page_alignment) { #if !MI_HUGE_PAGE_ABANDON mi_assert_internal(pq != NULL); mi_assert_internal(mi_theap_contains_queue(theap, pq)); mi_assert_internal(page_alignment > 0 || block_size > MI_LARGE_MAX_OBJ_SIZE || block_size == pq->block_size); #endif - mi_page_t* page = _mi_arenas_page_alloc(theap, block_size, page_alignment); - if (page == NULL) { - // out-of-memory - return NULL; - } - if (mi_page_is_abandoned(page)) { - _mi_theap_page_reclaim(theap, page); + mi_page_t* page; + bool reclaimed; + do { + page = _mi_arenas_page_alloc(theap, block_size, page_alignment); + if (page == NULL) { + // out-of-memory + return NULL; + } + reclaimed = mi_page_is_abandoned(page); + } while (reclaimed && _mi_theap_page_reclaim(theap, page)); + if (reclaimed) { if (!mi_page_immediate_available(page)) { if (mi_page_is_expandable(page)) { - if (!mi_page_extend_free(theap, page)) { + if (!_mi_page_extend_free(theap, page)) { // cannot commit _mi_page_abandon(page,pq); return NULL; @@ -521,9 +522,9 @@ static mi_page_t* mi_page_fresh_alloc(mi_theap_t* theap, mi_page_queue_t* pq, si } // Get a fresh page to use -static mi_page_t* mi_page_fresh(mi_theap_t* theap, mi_page_queue_t* pq) { +mi_page_t* _mi_page_fresh(mi_theap_t* theap, mi_page_queue_t* pq) { mi_assert_internal(mi_theap_contains_queue(theap, pq)); - mi_page_t* page = mi_page_fresh_alloc(theap, pq, pq->block_size, 0); + mi_page_t* page = _mi_page_fresh_alloc(theap, pq, pq->block_size, 0); if (page==NULL) return NULL; mi_assert_internal(pq->block_size==mi_page_block_size(page)); mi_assert_internal(pq==mi_theap_page_queue_of(theap, page)); @@ -551,7 +552,7 @@ void _mi_page_unfull(mi_page_t* page) { mi_page_queue_enqueue_from_full(pq, pqfull, page); } -static void mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) { +void _mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) { mi_assert_internal(pq == mi_page_queue_of(page)); mi_assert_internal(!mi_page_immediate_available(page)); mi_assert_internal(!mi_page_is_in_full(page)); @@ -565,22 +566,20 @@ static void mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) { // put full pages in a theap local queue (this is for theaps that cannot abandon, for example, if the theap can be destroyed) mi_page_queue_enqueue_from(&mi_page_theap(page)->pages[MI_BIN_FULL], pq, page); _mi_page_free_collect(page, false); // try to collect right away in case another thread freed just before MI_USE_DELAYED_FREE was set - _mi_page_update_stats(page); // we must update stats here as mi_theap_collect does not normally visit full pages + _mi_page_update_stats(page); // we must update stats here as mi_theap_collect does not normally visit full pages } } -// Free a page with no more free blocks -void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq) { +// Physically release an empty page after its grace period has finished. +static void mi_page_free_prim(mi_page_t* page, mi_page_queue_t* pq) { mi_assert_internal(page != NULL); + mi_assert_internal(_mi_page_qsbr_is_clear(page)); mi_assert_expensive(_mi_page_is_valid(page)); mi_assert_internal(pq == mi_page_queue_of(page)); mi_assert_internal(mi_page_all_free(page)); // mi_assert_internal(mi_page_thread_free_flag(page)!=MI_DELAYED_FREEING); - // no more aligned blocks in here - mi_page_set_has_interior_pointers(page, false); - // remove from the page list // (no need to do _mi_theap_delayed_free first as all blocks are already free) mi_page_queue_remove(pq, page); @@ -592,6 +591,30 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq) { // _mi_arenas_collect(false, false, theap->tld); // allow purging } +// Release an empty page (or retain it until its grace period has finished). +bool _mi_page_free(mi_page_t* page, mi_page_queue_t* pq) { + mi_assert_internal(mi_page_all_free(page)); + mi_assert_internal(pq == mi_page_queue_of(page)); + + // no more aligned blocks in here + mi_page_set_has_interior_pointers(page, false); + + // try to free + if (_mi_page_qsbr_should_delay(page)) { + // for pages that are delayed due to qsbr (e.g. in CPython) + page->retire_expire = 0; + if (mi_page_is_in_full(page)) { _mi_page_unfull(page); } + mi_page_qsbr_enqueue(page, mi_page_theap(page)); + return false; + } + else { + // free the page + _mi_page_qsbr_clear(page); + mi_page_free_prim(page, pq); + return true; + } +} + #define MI_RETIRE_CYCLES (16) /* keep a retired page around for about 16 "admin cycles" before free'ing it */ #define MI_RETIRE_MAX_PAGES (3) /* keep at most N pages per size bin as retired */ @@ -631,7 +654,7 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { if (index > theap->page_retired_max) theap->page_retired_max = index; mi_assert_internal(mi_page_all_free(page)); return; // don't free after all - } + } } #endif _mi_page_free(page, pq); @@ -659,7 +682,7 @@ static void mi_theap_collect_full_pages(mi_theap_t* theap) { } } -static void mi_page_try_retire(mi_page_queue_t* pq, mi_page_t* page, size_t bin, bool force, size_t* min, size_t* max) { +static void mi_page_try_retire(mi_page_queue_t* pq, mi_page_t* page, size_t bin, bool force, size_t* min, size_t* max) { mi_assert_internal(page!=NULL && page->retire_expire!=0); if (mi_page_all_free(page)) { page->retire_expire--; @@ -674,7 +697,7 @@ static void mi_page_try_retire(mi_page_queue_t* pq, mi_page_t* page, size_t bin, } else { page->retire_expire = 0; - } + } } // free retired pages: we don't need to look at the entire queues @@ -696,6 +719,7 @@ void _mi_theap_collect_retired(mi_theap_t* theap, bool force) { if (!theap->allow_page_abandon) { mi_theap_collect_full_pages(theap); } + _mi_theap_qsbr_collect_delayed(theap); } @@ -744,11 +768,11 @@ static void mi_page_free_list_extend_secure(mi_theap_t* const theap, mi_page_t* counts[current]--; mi_block_t* const free_start = blocks[current]; // and iterate through the rest; use `random_shuffle` for performance - size_t rnd = _mi_random_shuffle(r|1); // ensure not 0 + size_t rnd = mi_random_shuffle(r|1); // ensure not 0 for (size_t i = 1; i < extend; i++) { // call random_shuffle only every SIZE_SIZE rounds const size_t round = i%MI_SIZE_SIZE; - if (round == 0) rnd = _mi_random_shuffle(rnd); + if (round == 0) rnd = mi_random_shuffle(rnd); // select a random next slice index size_t next = ((rnd >> 8*round) & (slice_count-1)); while (counts[next]==0) { // ensure it still has space @@ -808,7 +832,7 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, co // Note: we also experimented with "bump" allocation on the first // allocations but this did not speed up any benchmark (due to an // extra test in malloc? or cache effects?) -static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) { +bool _mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) { mi_assert_expensive(mi_page_is_valid_init(page)); #if (MI_SECURE < 2) mi_assert(page->free == NULL); @@ -821,7 +845,7 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) { //uint8_t* page_start = mi_page_area(page, &page_size); mi_theap_stat_counter_increase(theap, pages_extended, 1); - + // calculate the extend count const size_t bsize = mi_page_block_size(page); size_t extend = (size_t)page->reserved - page->capacity; @@ -845,22 +869,22 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) { if (slice_committed > 0) { // reduce extend if it commits more than an arena slice if ((extend * bsize) > MI_ARENA_SLICE_SIZE) { - extend = _mi_divide_up(MI_ARENA_SLICE_SIZE, bsize); + extend = mi_divide_up(MI_ARENA_SLICE_SIZE, bsize); } // commit required size const size_t needed_size = (page->capacity + extend)*bsize; mi_assert_internal(needed_size <= page_size); - size_t needed_commit = _mi_align_up( mi_page_slice_offset_of(page, needed_size), mi_page_min_commit_size()); + size_t needed_commit = mi_align_up( mi_page_slice_offset_of(page, needed_size), mi_page_min_commit_size()); #if MI_SECURE>=5 // the previous alignup could extend the commit into the guard page; re-adjust if needed - const size_t page_size_commit = _mi_align_up( mi_page_slice_offset_of(page, page_size), _mi_os_page_size() ); - if (needed_commit > page_size_commit) { + const size_t page_size_commit = mi_align_up( mi_page_slice_offset_of(page, page_size), _mi_os_page_size() ); + if (needed_commit > page_size_commit) { needed_commit = page_size_commit; } #endif if (needed_commit > slice_committed) { mi_assert_internal(((needed_commit - slice_committed) % _mi_os_page_size()) == 0); - if (!_mi_os_commit(_mi_theap_subproc(theap), mi_page_slice_start(page) + slice_committed, needed_commit - slice_committed, NULL)) { + if (!_mi_os_commit(mi_theap_subproc(theap), mi_page_slice_start(page) + slice_committed, needed_commit - slice_committed, NULL)) { return false; } mi_assert_internal(needed_commit <= UINT16_MAX * _mi_os_page_size()); @@ -886,7 +910,7 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) { mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) { mi_assert(page != NULL); mi_assert(theap!=NULL); - // page->heap = (_mi_is_heap_main(_mi_theap_heap(theap)) ? NULL : _mi_theap_heap(theap)); // faster for `mi_page_associated_theap` + // page->heap = (mi_is_heap_main(mi_theap_heap(theap)) ? NULL : mi_theap_heap(theap)); // faster for `mi_page_associated_theap` // mi_page_set_theap(page, theap); size_t page_size; @@ -908,7 +932,7 @@ mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) { #endif mi_assert_internal(page->heap != NULL); - mi_assert_internal(page->heap == _mi_theap_heap(theap)); + mi_assert_internal(page->heap == mi_theap_heap(theap)); mi_assert_internal(page->theap!=NULL); mi_assert_internal(page->theap == mi_page_theap(page)); mi_assert_internal(page->capacity == 0); @@ -930,442 +954,7 @@ mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) { mi_assert_expensive(mi_page_is_valid_init(page)); // initialize an initial free list - if (!mi_page_extend_free(theap,page)) return false; + if (!_mi_page_extend_free(theap,page)) return false; mi_assert(mi_page_immediate_available(page)); return true; } - - -/* ----------------------------------------------------------- - Find pages with free blocks --------------------------------------------------------------*/ - -// Find a page with free blocks of `page->block_size`. -static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, mi_page_queue_t* pq, bool first_try) -{ - // search through the pages in "next fit" order - size_t count = 0; - long candidate_limit = 0; // we reset this on the first candidate to limit the search - long page_full_retain = (pq->block_size > MI_SMALL_MAX_OBJ_SIZE ? 0 : theap->page_full_retain); // only retain small pages - mi_page_t* page_candidate = NULL; // a page with free space - mi_page_t* page = pq->first; - mi_page_t* const last = pq->last; - - while (page!=NULL) - { - mi_page_t* next = page->next; // remember next (as this page can move to another queue) - count++; - candidate_limit--; - - // search up to N pages for a best candidate - - // is the local free list non-empty? - bool immediate_available = mi_page_immediate_available(page); - if (!immediate_available) { - // collect freed blocks by us and other threads to we get a proper use count - _mi_page_free_collect(page, false); - immediate_available = mi_page_immediate_available(page); - } - - // if the page is completely full, move it to the `mi_pages_full` - // queue so we don't visit long-lived pages too often. - if (!immediate_available && !mi_page_is_expandable(page)) { - page_full_retain--; - if (page_full_retain < 0) { - mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page)); - mi_page_to_full(page, pq); - } - else if (page!=last && pq->last!=page) { - // avoid revisiting this page for a while - mi_page_queue_move_to_back(theap, pq, page); - } - } - else { - // the page has free space, make it a candidate - // we prefer non-expandable pages with high usage as candidates (to reduce commit, and increase chances of free-ing up pages) - if (page_candidate == NULL) { - page_candidate = page; - candidate_limit = _mi_option_get_fast(mi_option_page_max_candidates); - } - else if (mi_page_all_free(page_candidate)) { - _mi_page_free(page_candidate, pq); - page_candidate = page; - } - // prefer to reuse fuller pages (in the hope the less used page gets freed) - else if (mi_page_used(page) >= mi_page_used(page_candidate) && !mi_page_is_mostly_used(page)) { // && !mi_page_is_expandable(page)) { - page_candidate = page; - } - // if we find a non-expandable candidate, or searched for N pages, return with the best candidate - if (immediate_available || candidate_limit <= 0) { - mi_assert_internal(page_candidate!=NULL); - break; - } - } - - #if 0 - // first-fit algorithm without candidates - // If the page contains free blocks, we are done - if (mi_page_immediate_available(page) || mi_page_is_expandable(page)) { - break; // pick this one - } - - // If the page is completely full, move it to the `mi_pages_full` - // queue so we don't visit long-lived pages too often. - mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page)); - mi_page_to_full(page, pq); - #endif - if (page==last) { page=NULL; } // don't revisit earlier pages that moved to the back - else { page = next; } - } // for each page - - mi_theap_stat_counter_increase(theap, page_searches, count); - mi_theap_stat_counter_increase(theap, page_searches_count, 1); - - // set the page to the best candidate - if (page_candidate != NULL) { - page = page_candidate; - } - if (page != NULL) { - if (!mi_page_immediate_available(page)) { - mi_assert_internal(mi_page_is_expandable(page)); - if (!mi_page_extend_free(theap, page)) { - page = NULL; // failed to extend - } - } - mi_assert_internal(page == NULL || mi_page_immediate_available(page)); - } - - if (page == NULL) { - _mi_theap_collect_retired(theap, false); // perhaps make a page available - page = mi_page_fresh(theap, pq); - mi_assert_internal(page == NULL || mi_page_immediate_available(page)); - if (page == NULL && first_try) { - // out-of-memory _or_ an abandoned page with free blocks was reclaimed, try once again - page = mi_page_queue_find_free_ex(theap, pq, false); - mi_assert_internal(page == NULL || mi_page_immediate_available(page)); - } - } - else { - mi_assert_internal(page == NULL || mi_page_immediate_available(page)); - // move the page to the front of the queue - mi_page_queue_move_to_front(theap, pq, page); - page->retire_expire = 0; - // _mi_theap_collect_retired(theap, false); // update retire counts; note: increases rss on MemoryLoad bench so don't do this - } - mi_assert_internal(page == NULL || mi_page_immediate_available(page)); - - - return page; -} - -// Look for a page with free blocks of `size` but don't try to search or allocate -static inline mi_page_t* mi_page_queue_lookup_free_first(mi_theap_t* theap, mi_page_queue_t* pq) { - mi_assert_internal(!mi_page_queue_is_huge(pq)); - mi_page_t* page = pq->first; - if mi_likely(page!=NULL && mi_page_free_quick_collect(page)) { - // fast path - #if (MI_SECURE>=2) // in secure mode, we extend half the time to increase randomness - if (page->capacity < page->reserved && ((_mi_theap_random_next(theap) & 1) == 1)) { - (void)mi_page_extend_free(theap, page); // ok if this fails - mi_assert_internal(mi_page_immediate_available(page)); - } - #else - MI_UNUSED(theap); - #endif - page->retire_expire = 0; - mi_assert_internal(mi_page_immediate_available(page)); - mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); - mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); - return page; - } - else { - return NULL; - } -} - -// Find a page with free blocks of `size`. -static inline mi_page_t* mi_page_queue_find_free(mi_theap_t* theap, mi_page_queue_t* pq) { - // mi_page_queue_t* pq = mi_page_queue(theap, size); - mi_assert_internal(!mi_page_queue_is_huge(pq)); - - // check the first page: we even do this with candidate search or otherwise we re-search every time - mi_page_t* page = mi_page_queue_lookup_free_first(theap,pq); - if (page==NULL) { - page = mi_page_queue_find_free_ex(theap, pq, true); - if (page==NULL) return NULL; - } - mi_assert_internal(mi_page_immediate_available(page)); - mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); - mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); - return page; -} - -// Huge pages contain just one block, and the segment contains just that page. -// Huge pages are also use if the requested alignment is very large (> MI_BLOCK_ALIGNMENT_MAX) -// so their size is not always `> MI_LARGE_OBJ_SIZE_MAX`. -static mi_page_t* mi_huge_page_alloc(mi_theap_t* theap, size_t size, size_t page_alignment, mi_page_queue_t* pq) { - const size_t block_size = _mi_os_good_alloc_size(size); - // mi_assert_internal(mi_bin(block_size) == MI_BIN_HUGE || page_alignment > 0); - #if MI_HUGE_PAGE_ABANDON - #error todo. - #else - // mi_page_queue_t* pq = mi_page_queue(theap, MI_LARGE_MAX_OBJ_SIZE+1); // always in the huge queue regardless of the block size - mi_assert_internal(mi_page_queue_is_huge(pq)); - #endif - mi_page_t* page = mi_page_fresh_alloc(theap, pq, block_size, page_alignment); - if (page != NULL) { - mi_assert_internal(mi_page_block_size(page) >= size); - mi_assert_internal(mi_page_immediate_available(page)); - mi_assert_internal(mi_page_is_huge(page)); - mi_assert_internal(mi_page_is_singleton(page)); - #if MI_HUGE_PAGE_ABANDON - mi_assert_internal(mi_page_is_abandoned(page)); - mi_page_set_theap(page, NULL); - #endif - // mi_theap_stat_increase(theap, malloc_huge, mi_page_block_size(page)); - // mi_theap_stat_counter_increase(theap, malloc_huge_count, 1); - } - return page; -} - -// Allocate a page -// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed. -static mi_page_t* mi_find_page(mi_theap_t* theap, size_t size, size_t huge_alignment) mi_attr_noexcept { - const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` - if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE) { // TODO: remove as we check in generic_fallback ? - _mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size); - return NULL; - } - mi_page_queue_t* pq = mi_page_queue(theap, (huge_alignment > 0 ? MI_LARGE_MAX_OBJ_SIZE+1 : size)); - mi_page_t* page; - // huge allocation? - if mi_unlikely(mi_page_queue_is_huge(pq) || req_size > MI_MAX_ALLOC_SIZE) { - page = mi_huge_page_alloc(theap,size,huge_alignment,pq); - } - else { - // otherwise find a page with free blocks in our size segregated queues - #if MI_PADDING - mi_assert_internal(size >= MI_PADDING_SIZE); - #endif - page = mi_page_queue_find_free(theap,pq); - } - if (page==NULL) return NULL; - mi_assert_internal(mi_page_block_size(page) >= size); - mi_assert_internal(mi_page_immediate_available(page)); - mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); - mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); - return page; -} - - - -/* ----------------------------------------------------------- - Users can register a deferred free function called - when the `free` list is empty. Since the `local_free` - is separate this is deterministically called after - a certain number of allocations. ------------------------------------------------------------ */ - -// The program should only install a single deferred free handler before doing allocation. -static _Atomic(void*) deferred_free; // is `mi_deferred_free_fun*` (but some platforms don't support atomic function pointers) -static _Atomic(void*) deferred_arg; - -void _mi_deferred_free(mi_theap_t* theap, bool force) { - theap->heartbeat++; - mi_deferred_free_fun* const fun = (mi_deferred_free_fun*)mi_atomic_load_ptr_acquire(void,&deferred_free); - if (fun != NULL && !theap->tld->recurse) { - theap->tld->recurse = true; - void* const arg = mi_atomic_load_ptr_acquire(void,&deferred_arg); - fun(force, theap->heartbeat, arg); - theap->tld->recurse = false; - } -} - -void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noexcept { - mi_atomic_store_ptr_release(void,&deferred_arg, arg); - mi_atomic_store_ptr_release(void,&deferred_free, (void*)fn); -} - - -/* ----------------------------------------------------------- - Admin ------------------------------------------------------------ */ -static mi_theap_t* mi_theap_init(mi_theap_t* theap) { - if mi_likely(mi_theap_is_initialized(theap)) return theap; - if (theap==&_mi_theap_empty_wrong) { - // we were unable to allocate a theap for a first-class heap - return NULL; - } - // otherwise we initialize the thread and its default theap - theap = _mi_thread_init(); - if mi_unlikely(!mi_theap_is_initialized(theap)) { return NULL; } - mi_assert_internal(mi_theap_is_initialized(theap)); - return theap; -} - -static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap) -{ - theap = mi_theap_init(theap); - if (theap==NULL) return NULL; - mi_assert_internal(mi_theap_is_initialized(theap)); - - // do administrative tasks every N generic mallocs - if mi_unlikely(theap->generic_count >= 1000) { - theap->generic_collect_count += theap->generic_count; - theap->generic_count = 0; - - // check if the profiler is enabled - mi_heap_t* const heap = _mi_theap_heap(theap); - mi_profiler_t* prof = mi_atomic_load_ptr_relaxed(mi_profiler_t, &heap->profiler); - const bool prof_enabled = (prof!=NULL && mi_profiler_is_enabled(prof)); - if (prof_enabled && theap->profile_sample_rate==0) { - _mi_theap_set_profile_sample_rate(theap,mi_max(1,prof->initial_sample_rate)); // start profiling - } - else if (!prof_enabled && theap->profile_sample_rate!=0) { - _mi_theap_set_profile_sample_rate(theap,0); // stop profiling - } - - // do a full theap collect every once in a while (10000 by default) - const long generic_collect = mi_option_get_clamp(mi_option_generic_collect, 1, 1000000L); - if (theap->generic_collect_count >= generic_collect) { - theap->generic_collect_count = 0; - mi_theap_collect(theap, false /* force? */); - } - // else if (theap->sample_rate != 0) { // update stats more aggressively if we are sampling - // mi_theap_collect(theap, false /* force? */); // TODO: make specialized mi_theap_collect_update_stats ? - // } - else { - // otherwise we do a mini-collect - _mi_deferred_free(theap, false); // call potential deferred free routines - _mi_theap_collect_retired(theap, false); // free retired pages - } - } - return theap; -} - -/* ----------------------------------------------------------- - Generic allocation ------------------------------------------------------------ */ - -static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) -{ - // initialize if necessary - theap = mi_malloc_generic_admin(theap); - if (theap==NULL) return NULL; - - // check allocation size - const size_t req_size = size - MI_PADDING_SIZE; - if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { - _mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size); - return NULL; - } - - // take a sample? - bool sample_countdown_is_adjusted = false; - if mi_unlikely(mi_theap_should_sample(theap,req_size)) { - if (huge_alignment==0 && theap->sample_rate!=0) { - return _mi_theap_malloc_sampled(theap,req_size,zero,ppage); - } - mi_assert_internal(!_mi_is_empty_theap(theap)); // cannot write to the empty theap - mi_assert_internal(req_size <= MI_SAMPLE_COUNTDOWN_MAX); - if (theap->sample_rate==0) { - theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // reset the countdown counter - } - #if MI_SAMPLE==2 - else { - // huge_alignment!=0 so we need to adjust the countdown - mi_assert_internal(huge_alignment!=0); - mi_assert_internal(theap->sample_countdown <= SIZE_MAX - req_size); - sample_countdown_is_adjusted = true; - theap->sample_countdown += req_size; // adjust such that after `_mi_page_malloc_zero` the countdown is correct again - } - #endif - } - - // find (or allocate) a page of the right size - mi_page_t* page = mi_find_page(theap, size, huge_alignment); - if mi_unlikely(page == NULL) { // first time out of memory, try to collect and retry the allocation once more - mi_theap_collect(theap, true /* force? */); - page = mi_find_page(theap, size, huge_alignment); - } - - if mi_unlikely(page == NULL) { // out of memory - if (sample_countdown_is_adjusted) { theap->sample_countdown -= req_size; } // unadjust again if we failed to find a page - _mi_error_message(ENOMEM, "unable to allocate memory (%zu bytes)\n", req_size); - return NULL; - } - - mi_assert_internal(mi_page_immediate_available(page)); - mi_assert_internal(mi_page_block_size(page) >= size); - mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); - mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); - - // and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc_zero) - if (ppage!=NULL) { *ppage = page; } - void* const p = _mi_page_malloc_zero(theap,page,size,zero); - mi_assert_internal(p != NULL); - mi_page_update_sample_countdown(page); - - // move full pages to the full queue - // this will also call _mi_page_update_stats for huge pages - if (mi_page_block_size(page) > MI_SMALL_MAX_OBJ_SIZE) { - if (mi_page_is_full(page)) { - mi_page_to_full(page, mi_page_queue_of(page)); - } - } - return p; -} - - -// Generic allocation routine if the fast path (`alloc.c:mi_page_malloc`) does not succeed. -// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed. -// The `huge_alignment` is normally 0 but is set to a multiple of MI_SLICE_SIZE for -// very large requested alignments in which case we use a huge singleton page. -// Note: we put `bool zero, size_t huge_alignment` into one parameter (with zero in the low bit) -// to use 4 parameters which compiles better on msvc for the malloc fast path. -void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept -{ - #if !MI_THEAP_INITASNULL - mi_assert_internal(theap != NULL); - #endif - const bool zero = ((zero_huge_alignment & 1) != 0); - const size_t huge_alignment = (zero_huge_alignment & ~1); - mi_page_t* page = NULL; - - // fast path objects that fit in a small page - if mi_likely(mi_theap_is_initialized(theap) && ++theap->generic_count < 1000 && huge_alignment==0) { - const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` - if (req_size < MI_SMALL_MAX_OBJ_SIZE) - { - #if MI_SAMPLE - if mi_likely(!mi_theap_should_sample(theap,req_size)) // ensure we don't need to take a sample - #endif - { - mi_page_queue_t* pq = mi_page_queue(theap, size); - mi_assert_internal(pq!=NULL && !mi_page_queue_is_huge(pq)); - page = mi_page_queue_find_free(theap,pq); - // mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); - if (page!=NULL) { - if (ppage!=NULL) { *ppage = page; } - mi_assert_internal(mi_page_immediate_available(page)); // we should never recurse in _mi_page_malloc_zero - return _mi_page_malloc_zero(theap,page,size,zero); - } - } - } - } - // otherwise fallback - return mi_malloc_generic_fallback(theap,size,zero,huge_alignment,ppage); -} - -void* _mi_malloc_generic_no_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { - theap = mi_theap_init(theap); - if (theap==NULL) return NULL; - const size_t sample_rate = theap->sample_rate; - const size_t sample_countdown = theap->sample_countdown; - theap->sample_rate = 0; // prevent a recursive call to mi_theap_malloc_sampled from _mi_malloc_generic - theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; - void* p = _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); - theap->sample_rate = sample_rate; - theap->sample_countdown = sample_countdown; - return p; -} diff --git a/system/lib/mimalloc/src/alloc/page.h b/system/lib/mimalloc/src/alloc/page.h new file mode 100644 index 0000000000000..ae2af1760b0f0 --- /dev/null +++ b/system/lib/mimalloc/src/alloc/page.h @@ -0,0 +1,37 @@ +/*---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_PAGE_H +#define MI_PAGE_H + +#include "mimalloc.h" +#include "page-queue.h" + +bool _mi_page_extend_free(mi_theap_t* theap, mi_page_t* page); +mi_page_t* _mi_page_fresh_alloc(mi_theap_t* theap, mi_page_queue_t* pq, size_t block_size, size_t page_alignment); +mi_page_t* _mi_page_fresh(mi_theap_t* theap, mi_page_queue_t* pq); +void _mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq); +void _mi_page_update_sample_countdown(mi_page_t* page); + +static inline void _mi_theap_qsbr_collect_delayed(mi_theap_t* theap) { +#if MI_QSBR + mi_page_t* page; + while ((page = theap->qsbr_first) != NULL) { + _mi_page_free_collect(page, false); + if (!mi_page_all_free(page)) { + _mi_page_qsbr_clear(page); + } + else if (!_mi_page_free(page, mi_page_queue_of(page)) && theap->qsbr_first == page) { + break; + } + } +#else + MI_UNUSED(theap); +#endif +} + +#endif diff --git a/system/lib/mimalloc/src/alloc/readme.md b/system/lib/mimalloc/src/alloc/readme.md new file mode 100644 index 0000000000000..0c354f70aae6a --- /dev/null +++ b/system/lib/mimalloc/src/alloc/readme.md @@ -0,0 +1,12 @@ +# Allocation and Freeing + +- [`alloc.c`](alloc.c): Main allocation and reallocation functions, including the allocation fast paths. +- [`alloc-generic.c`](alloc-generic.c): Allocation slow path (`mi_malloc_generic`), page selection, deferred-free callbacks. +- [`alloc-aligned.c`](alloc-aligned.c): Aligned allocation and reallocation, including allocations with an alignment offset. +- [`alloc-posix.c`](alloc-posix.c): POSIX, Unix, and Microsoft-compatible allocation interfaces and convenience functions. +- [`alloc-override.c`](alloc-override.c): Standard allocator and C++ allocation overrides, included by `alloc.c`. +- [`free.c`](free.c): Local and cross-thread freeing, pointer validation, and usable-size queries, included by `alloc.c`. +- [`page.c`](page.c): Allocation-page initialization, free-list management, collection, and retirement. +- [`page.h`](page.h): Private declarations for page helpers used by generic allocation. +- [`page-queue.c`](page-queue.c): Size-class mapping and management of page queues. +- [`page-queue.h`](page-queue.h): Private page-queue declarations and inline helpers. diff --git a/system/lib/mimalloc/src/arena.c b/system/lib/mimalloc/src/arena.c deleted file mode 100644 index 1b7c8cb336d0c..0000000000000 --- a/system/lib/mimalloc/src/arena.c +++ /dev/null @@ -1,2777 +0,0 @@ -/* ---------------------------------------------------------------------------- -Copyright (c) 2019-2026, Microsoft Research, Daan Leijen -This is free software; you can redistribute it and/or modify it under the -terms of the MIT license. A copy of the license can be found in the file -"LICENSE" at the root of this distribution. ------------------------------------------------------------------------------*/ - -/* ---------------------------------------------------------------------------- -"Arenas" are fixed area's of OS memory from which we can allocate -large blocks (>= MI_ARENA_MIN_BLOCK_SIZE, 64KiB). -In contrast to the rest of mimalloc, the arenas are shared between -threads and need to be accessed using atomic operations. - -Arenas are also used to for huge OS page (1GiB) reservations or for reserving -OS memory upfront which can be improve performance or is sometimes needed -on embedded devices. We can also employ this with WASI or `sbrk` systems -to reserve large arenas upfront and be able to reuse the memory more effectively. - -The arena allocation needs to be thread safe and we use an atomic bitmap to allocate. ------------------------------------------------------------------------------*/ - -#include "mimalloc.h" -#include "mimalloc/internal.h" -#include "mimalloc/prim-tls.h" -#include "bitmap.h" - - -/* ----------------------------------------------------------- - Arena id's ------------------------------------------------------------ */ - -mi_arena_id_t _mi_arena_id_none(void) { - return NULL; -} - -mi_arena_t* _mi_arena_from_id(mi_arena_id_t id) { - mi_arena_t* const arena = (mi_arena_t*)id; - mi_assert_internal(arena==NULL || arena->parent==NULL); // id's should never point to sub-arena's - return arena; -} - -mi_arena_id_t mi_arena_id_from_arena(mi_arena_t* arena) { - mi_assert_internal(arena==NULL || arena->parent==NULL); - return (arena==NULL ? _mi_arena_id_none() : (mi_arena_id_t)arena); -} - - -static bool mi_arena_is_suitable(mi_arena_t* arena, mi_arena_t* req_arena) { - if (arena == req_arena) return true; // they match - if (arena == NULL) return false; - if (req_arena == NULL && !arena->is_exclusive) return true; // or the arena is not exclusive, and we didn't request a specific one - if (arena->parent != NULL && arena->parent == req_arena) return true; // sub-arena? (note that req_arena is never a sub arena) - return false; -} - -bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_t* request_arena) { - if (memid.memkind == MI_MEM_ARENA) { - return mi_arena_is_suitable(memid.mem.arena.arena, request_arena); - } - else { - return mi_arena_is_suitable(NULL, request_arena); - } -} - -size_t mi_arenas_get_count(mi_subproc_t* subproc) { - return mi_atomic_load_relaxed(&subproc->arena_count); -} - -mi_arena_t* mi_arena_from_index(mi_subproc_t* subproc, size_t idx) { - mi_assert_internal(idx < mi_arenas_get_count(subproc)); - return mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[idx]); -} - -static size_t mi_arena_info_slices(mi_arena_t* arena) { - return arena->info_slices; -} - -#if MI_DEBUG > 1 -static bool mi_heap_has_page(mi_heap_t* heap, mi_arena_t* arena, mi_page_t* page) { - mi_assert(arena->arena_idx < MI_MAX_ARENAS); - mi_arena_pages_t* arena_pages = heap->arena_pages[arena->arena_idx]; - return (page->memid.memkind == MI_MEM_ARENA && - page->memid.mem.arena.arena == arena && - arena_pages != NULL && - mi_bitmap_is_setN(arena_pages->pages, page->memid.mem.arena.slice_index, 1)); -} -#endif - -size_t mi_arena_min_alignment(void) { - return MI_ARENA_ALIGNMENT; -} - -size_t mi_arena_min_size(void) { - return MI_ARENA_MIN_SIZE; -} - -// slices reserved for page meta info at the start of aligned chunks -static size_t mi_arena_page_meta_aligned_slice_count(void) { - #if MI_PAGE_META_IS_ALIGNED - return _mi_divide_up(MI_PAGE_META_ALIGNED_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE); - #else - return 0; - #endif -} - -// fixed limit for the maximum object size in an arena -static size_t mi_arena_max_fixed_object_size(void) { - #if MI_PAGE_META_IS_ALIGNED - return (MI_PAGE_META_ALIGNMENT - _mi_align_up(MI_PAGE_META_ALIGNED_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE)); - #else - return (MI_ARENA_MAX_SIZE - MI_ARENA_CHUNK_SIZE); // minus an initial chunk to accommodate meta info - #endif -} - -// Maximum object size allowed to be allocated in an arena -size_t mi_arena_max_object_size(void) { - size_t max_size = mi_option_get_size(mi_option_arena_max_object_size); - max_size = _mi_align_up(max_size, MI_ARENA_SLICE_SIZE); - if (max_size <= MI_ARENA_MIN_OBJ_SIZE) { - return MI_ARENA_MIN_OBJ_SIZE; - } - else if (max_size >= mi_arena_max_fixed_object_size()) { - return mi_arena_max_fixed_object_size(); - } - else { - return max_size; - } -} - -mi_decl_nodiscard static bool mi_arena_commit(mi_subproc_t* subproc, mi_arena_t* arena, void* start, size_t size, bool* is_zero, size_t stat_already_committed) { - mi_assert_internal(subproc!=NULL); - if (arena != NULL && arena->commit_fun != NULL) { - return (*arena->commit_fun)(true, start, size, is_zero, arena->commit_fun_arg); - } - else if (stat_already_committed > 0) { - return _mi_os_commit_ex(subproc, start, size, is_zero, stat_already_committed); - } - else { - return _mi_os_commit(subproc, start, size, is_zero); - } -} - - - -/* ----------------------------------------------------------- - Util ------------------------------------------------------------ */ - - -// Size of an arena -static size_t mi_arena_size(mi_arena_t* arena) { - return mi_size_of_slices(arena->slice_count); -} - -// Start of the arena memory area -static uint8_t* mi_arena_start(mi_arena_t* arena) { - return ((uint8_t*)arena->start); -} - -// Start of a slice -uint8_t* mi_arena_slice_start(mi_arena_t* arena, size_t slice_index) { - mi_assert_internal(slice_index < arena->slice_count); - return (mi_arena_start(arena) + mi_size_of_slices(slice_index)); -} - -mi_page_t* mi_arena_page_at_slice(mi_arena_t* arena, size_t slice_index) { - mi_assert_internal(slice_index < arena->slice_count); - #if MI_PAGE_META_IS_ALIGNED - mi_page_t* const page = _mi_aligned_ptr_page(mi_arena_slice_start(arena,slice_index)); // todo: optimize? - return page; - #else - if (arena->pages_meta != NULL) { - mi_page_t* const page = &arena->pages_meta[slice_index]; - #if MI_PAGE_META_SMALL_IS_ALIGNED - // pages with small blocks still have the page at the start of the slice (and set the `block_size` in pages_meta to 0) - if (page->block_size>0) return page; - #else - return page; - #endif - } - // fall through (for MI_PAGE_META_SMALL_IS_ALIGNED) - return (mi_page_t*)mi_arena_slice_start(arena,slice_index); - #endif -} - -// Arena area -void* mi_arena_area(mi_arena_id_t arena_id, size_t* size) { - if (size != NULL) *size = 0; - mi_arena_t* arena = _mi_arena_from_id(arena_id); - if (arena == NULL) return NULL; - if (size != NULL) { - mi_assert_internal(mi_size_of_slices(arena->slice_count) <= arena->total_size); - *size = arena->total_size; - } - return mi_arena_start(arena); -} - - -// Create an arena memid -static mi_memid_t mi_memid_create_arena(mi_arena_t* arena, size_t slice_index, size_t slice_count) { - mi_assert_internal(slice_index < UINT32_MAX); - mi_assert_internal(slice_count < UINT32_MAX); - mi_assert_internal(slice_count > 0); - mi_assert_internal(slice_index < arena->slice_count); - mi_memid_t memid = _mi_memid_create(MI_MEM_ARENA); - memid.mem.arena.arena = arena; - memid.mem.arena.slice_index = (uint32_t)slice_index; - memid.mem.arena.slice_count = (uint32_t)slice_count; - return memid; -} - -// get the arena and slice span -static mi_arena_t* mi_arena_from_memid(mi_memid_t memid, size_t* slice_index, size_t* slice_count) { - mi_assert_internal(memid.memkind == MI_MEM_ARENA); - mi_arena_t* arena = memid.mem.arena.arena; - if (slice_index!=NULL) { *slice_index = memid.mem.arena.slice_index; } - if (slice_count!=NULL) { *slice_count = memid.mem.arena.slice_count; } - return arena; -} - -static size_t mi_page_full_size(mi_page_t* page) { - if (page->memid.memkind == MI_MEM_ARENA) { - return page->memid.mem.arena.slice_count * MI_ARENA_SLICE_SIZE; - } - else if (mi_memid_is_os(page->memid) || page->memid.memkind == MI_MEM_EXTERNAL) { - mi_assert_internal((uint8_t*)page->memid.mem.os.base <= (uint8_t*)page); - const ptrdiff_t presize = (uint8_t*)page - (uint8_t*)page->memid.mem.os.base; - mi_assert_internal((ptrdiff_t)page->memid.mem.os.size >= presize); - return (presize > (ptrdiff_t)page->memid.mem.os.size ? 0 : page->memid.mem.os.size - presize); - } - else { - return 0; - } -} - -/* ----------------------------------------------------------- - Arena Allocation ------------------------------------------------------------ */ - -static mi_decl_noinline void* mi_arena_try_alloc_at( - mi_arena_t* arena, size_t slice_count, bool commit, size_t tseq, mi_memid_t* memid) -{ - mi_assert_internal(arena!=NULL); - mi_assert_internal(slice_count>0); - size_t slice_index; - if (!mi_bbitmap_try_find_and_clearN(arena->slices_free, tseq, slice_count, &slice_index)) return NULL; - - // claimed it! - void* p = mi_arena_slice_start(arena, slice_index); - *memid = mi_memid_create_arena(arena, slice_index, slice_count); - memid->is_pinned = arena->memid.is_pinned; - - // set the dirty bits and track which slices become accessible - size_t touched_slices = slice_count; - if (arena->memid.initially_zero) { - size_t already_dirty = 0; - memid->initially_zero = mi_bitmap_setN(arena->slices_dirty, slice_index, slice_count, &already_dirty); - mi_assert_internal(already_dirty <= touched_slices); - touched_slices -= already_dirty; - } - else { - // todo: properly count touched pages with a separate bitmap? - touched_slices = 0; - } - - // set commit state - if (commit) { - // commit requested, but the range may not be committed as a whole: ensure it is committed now - const size_t already_committed = mi_bitmap_popcountN(arena->slices_committed, slice_index, slice_count); - if (already_committed < slice_count) { - // not all committed, try to commit now - bool commit_zero = false; - if (!mi_arena_commit(arena->subproc, arena, p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(already_committed))) { - // if the commit fails, release ownership, and return NULL; - // note: this does not roll back dirty bits but that is ok. - mi_bbitmap_setN(arena->slices_free, slice_index, slice_count); - return NULL; - } - if (commit_zero) { - memid->initially_zero = true; - } - - // set the commit bits - mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, NULL); - - // committed - #if MI_DEBUG > 1 - if (memid->initially_zero) { - if (!mi_mem_is_zero(p, mi_size_of_slices(slice_count))) { - _mi_error_message(EFAULT, "internal error: arena allocation was not zero-initialized!\n"); - memid->initially_zero = false; - } - } - #endif - } - else { - // already fully committed. - _mi_os_reuse(arena->subproc, p, mi_size_of_slices(slice_count)); - // if the OS has overcommit, and this is the first time we access these pages, then - // count the commit now (as at arena reserve we didn't count those commits as these are on-demand) - if (_mi_os_has_overcommit() && touched_slices > 0 && !arena->memid.is_pinned /* huge pages, issue #1236 */) { - mi_subproc_stat_increase( arena->subproc, committed, mi_size_of_slices(touched_slices)); - } - } - - mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); - memid->initially_committed = true; - - // tool support - if (memid->initially_zero) { - mi_track_mem_defined(p, slice_count * MI_ARENA_SLICE_SIZE); - } - else { - mi_track_mem_undefined(p, slice_count * MI_ARENA_SLICE_SIZE); - } - } - else { - // no need to commit, but check if it is already fully committed - memid->initially_committed = mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count); - if (!memid->initially_committed) { - // partly committed.. adjust stats - size_t already_committed_count = 0; - mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed_count); - mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count); - mi_subproc_stat_decrease(arena->subproc, committed, mi_size_of_slices(already_committed_count)); - } - } - - mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); - if (commit) { mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); } - if (commit) { mi_assert_internal(memid->initially_committed); } - mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count)); - - return p; -} - - -static int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id); - -// try to reserve a fresh arena space -static bool mi_arena_reserve(mi_subproc_t* subproc, size_t req_size, bool allow_large, mi_arena_id_t* arena_id) -{ - const size_t arena_count = mi_arenas_get_count(subproc); - if (arena_count > (MI_MAX_ARENAS - 4)) return false; - - // calc reserve - size_t arena_reserve = mi_option_get_size(mi_option_arena_reserve); - if (arena_reserve == 0) return false; - - if (!_mi_os_has_virtual_reserve()) { - arena_reserve = arena_reserve/4; // be conservative if virtual reserve is not supported (for WASM for example) - } - arena_reserve = _mi_align_up(arena_reserve, MI_ARENA_SLICE_SIZE); - - if (arena_count >= 1 && arena_count <= 128) { - // scale up the arena sizes exponentially every 8 entries - const size_t multiplier = (size_t)1 << _mi_clamp(arena_count/8, 0, 16); - size_t reserve = 0; - if (!mi_mul_overflow(multiplier, arena_reserve, &reserve)) { - arena_reserve = reserve; - } - } - - // try to accommodate the requested size for huge allocations - req_size = _mi_align_up(req_size + MI_ARENA_MAX_CHUNK_OBJ_SIZE, MI_ARENA_MAX_CHUNK_OBJ_SIZE); // over-reserve for meta-info - if (arena_reserve < req_size) { - arena_reserve = req_size; - } - - // check arena bounds - const size_t min_reserve = MI_ARENA_MIN_SIZE; - const size_t max_reserve = MI_ARENA_MAX_SIZE; // 16 GiB - if (arena_reserve < min_reserve) { - arena_reserve = min_reserve; - } - else if (arena_reserve > max_reserve) { - arena_reserve = max_reserve; - } - - // should be able to at least handle the current allocation size - if (arena_reserve < req_size) return false; - - // commit eagerly? - bool arena_commit = false; - const bool overcommit = _mi_os_has_overcommit(); - if (mi_option_get(mi_option_arena_eager_commit) == 2) { arena_commit = overcommit || mi_option_is_enabled(mi_option_allow_large_os_pages); } - else if (mi_option_get(mi_option_arena_eager_commit) == 1) { arena_commit = true; } - - // on an OS with overcommit (Linux) we don't count the commit yet as it is on-demand. Once a slice - // is actually allocated for the first time it will be counted. - const bool adjust = (overcommit && arena_commit); - if (adjust) { mi_subproc_stat_adjust_decrease( subproc, committed, arena_reserve); } - // and try to reserve the arena - int err = mi_reserve_os_memory_ex2(subproc, arena_reserve, arena_commit, allow_large, false /* exclusive? */, arena_id); - if (err != 0) { - if (adjust) { mi_subproc_stat_adjust_increase( subproc, committed, arena_reserve); } // roll back - // failed to allocate: try a smaller size arena as fallback? - const size_t small_arena_reserve = 4 * MI_ARENA_MIN_SIZE; // 128 MiB (or 32 MiB on 32-bit) - if (arena_reserve > small_arena_reserve && small_arena_reserve > req_size) { - // try again - if (adjust) { mi_subproc_stat_adjust_decrease(subproc, committed, small_arena_reserve); } - err = mi_reserve_os_memory_ex2(subproc, small_arena_reserve, arena_commit, allow_large, false /* exclusive? */, arena_id); - if (err != 0 && adjust) { mi_subproc_stat_adjust_increase( subproc, committed, small_arena_reserve); } // roll back - } - } - return (err==0); -} - - - - -/* ----------------------------------------------------------- - Arena iteration ------------------------------------------------------------ */ - -static inline bool mi_arena_is_suitable_ex(mi_arena_t* arena, mi_arena_t* req_arena, bool match_numa, int numa_node, bool allow_pinned) { - if (!allow_pinned && arena->memid.is_pinned) return false; - if (!mi_arena_is_suitable(arena, req_arena)) return false; - if (req_arena == NULL) { // if not specific, check numa affinity - const bool numa_suitable = (numa_node < 0 || arena->numa_node < 0 || arena->numa_node == numa_node); - if (match_numa) { if (!numa_suitable) return false; } - else { if (numa_suitable) return false; } - } - return true; -} - -// determine the start of search; important to keep heaps and threads -// into their own memory regions to reduce contention. -static size_t mi_arena_start_idx(mi_heap_t* heap, size_t tseq, size_t arena_cycle) { - const size_t hseq = heap->heap_seq; - const size_t hcount = mi_atomic_load_relaxed(&heap->subproc->heap_count); - if (arena_cycle <= 1) return 0; - if (hseq==0 || hcount<=1 || arena_cycle > 0x8FF) return (tseq % arena_cycle); // common for single heap programs - - // spread heaps evenly among arena's, and then evenly for threads in their fraction - size_t start; - mi_assert_internal(arena_cycle <= 0x8FF); // prevent overflow on 32-bit - const size_t frac = (arena_cycle * 256) / hcount; // fraction in the arena_cycle; at most: arena_cycle * 0x100 - if (frac==0) { - // many heaps (> 256 per arena) - start = (hseq % arena_cycle); - } - else { - const size_t hspot = (hseq % hcount); - start = (frac * hspot) / 256; // (arena_cycle * (hseq % hcount)) / hcount - if (frac >= 512) { // at least 2 arena's per heap? - start = start + (tseq % (frac/256)); - } - } - mi_assert_internal(start < arena_cycle); - return start; -} - -#define mi_forall_arenas(heap, req_arena, tseq, name_arena) { \ - const size_t _arena_count = mi_arenas_get_count(heap->subproc); \ - const size_t _arena_cycle = (_arena_count == 0 ? 0 : _arena_count - 1); /* first search the arenas below the last one */ \ - /* always start searching in the arena's below the max */ \ - const size_t _start = (req_arena==NULL ? mi_arena_start_idx(heap,tseq,_arena_cycle) : ((mi_arena_t*)req_arena)->arena_idx); \ - mi_assert_internal(_start <= _arena_count); \ - for (size_t _i = 0; _i < _arena_count; _i++) { \ - mi_arena_t* name_arena; \ - size_t _idx; \ - if (_i < _arena_cycle) { \ - _idx = _i + _start; \ - if (_idx >= _arena_cycle) { _idx -= _arena_cycle; } /* adjust so we rotate through the cycle */ \ - } \ - else { \ - _idx = _i; /* remaining arena's after the cycle */ \ - } \ - name_arena = mi_arena_from_index(heap->subproc,_idx); \ - if (req_arena != NULL && name_arena != req_arena && \ - (name_arena == NULL || name_arena->parent != req_arena)) continue; /* only the requested arena or its children */ \ - if (name_arena != NULL) \ - { - -#define mi_forall_arenas_end() \ - } \ - } \ - } - -#define mi_forall_suitable_arenas(heap, req_arena, tseq, match_numa, numa_node, allow_large, name_arena) \ - mi_forall_arenas(heap, req_arena,tseq,name_arena) { \ - if (mi_arena_is_suitable_ex(name_arena, req_arena, match_numa, numa_node, allow_large)) { \ - -#define mi_forall_suitable_arenas_end() \ - }} \ - mi_forall_arenas_end() - -/* ----------------------------------------------------------- - Arena allocation ------------------------------------------------------------ */ - -// allocate slices from the arenas -static mi_decl_noinline void* mi_arenas_try_find_free( - mi_heap_t* heap, size_t slice_count, size_t alignment, - bool commit, bool allow_large, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid) -{ - // mi_assert_internal(slice_count <= mi_slice_count_of_size(MI_ARENA_MAX_CHUNK_OBJ_SIZE)); - mi_assert(alignment <= MI_ARENA_SLICE_ALIGN); - if (alignment > MI_ARENA_SLICE_ALIGN) return NULL; - - // search arena's - mi_forall_suitable_arenas(heap, req_arena, tseq, true /* only numa matching */, numa_node, allow_large, arena) - { - void* p = mi_arena_try_alloc_at(arena, slice_count, commit, tseq, memid); - if (p != NULL) return p; - } - mi_forall_suitable_arenas_end(); - if (numa_node < 0) return NULL; - - // search again but now regardless of preferred numa affinity - mi_forall_suitable_arenas(heap, req_arena, tseq, false /* numa non-matching now */, numa_node, allow_large, arena) - { - void* p = mi_arena_try_alloc_at(arena, slice_count, commit, tseq, memid); - if (p != NULL) return p; - } - mi_forall_suitable_arenas_end(); - return NULL; -} - -// Allocate slices from the arena's -- potentially allocating a fresh arena -static mi_decl_noinline void* mi_arenas_try_alloc( - mi_heap_t* heap, - size_t slice_count, size_t alignment, - bool commit, bool allow_large, - mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid) -{ - // mi_assert(slice_count <= MI_ARENA_MAX_CHUNK_OBJ_SLICES); - mi_assert(alignment <= MI_ARENA_SLICE_ALIGN); - void* p; - - // not too large? - if (slice_count * MI_ARENA_SLICE_SIZE > MI_ARENA_MAX_SIZE) return NULL; - - // try to find free slices in the arena's - p = mi_arenas_try_find_free(heap, slice_count, alignment, commit, allow_large, req_arena, tseq, numa_node, memid); - if (p != NULL) return p; - - // did we need a specific arena? - if (req_arena != NULL) return NULL; - - // don't create arena's while preloading (todo: or should we?) - // if (_mi_preloading()) return NULL; - - // don't create arena's if OS allocation is disallowed - if (mi_option_is_enabled(mi_option_disallow_os_alloc)) return NULL; - - // otherwise, try to reserve a new arena -- but one thread at a time.. (todo: allow 2 or 4 to reduce contention?) - mi_subproc_t* const subproc = heap->subproc; - const size_t arena_count = mi_arenas_get_count(subproc); - mi_lock(&subproc->arena_reserve_lock) { - if (arena_count == mi_arenas_get_count(subproc)) { - // we are the first to enter the lock, reserve a fresh arena - mi_arena_id_t arena_id = _mi_arena_id_none(); - mi_arena_reserve(subproc, mi_size_of_slices(slice_count), allow_large, &arena_id); - } - else { - // another thread already reserved a new arena - } - } - // try once more to allocate in the new arena - mi_assert_internal(req_arena == NULL); - p = mi_arenas_try_find_free(heap, slice_count, alignment, commit, allow_large, req_arena, tseq, numa_node, memid); - if (p != NULL) return p; - - return NULL; -} - -// Allocate from the OS (if allowed) -static void* mi_arena_os_alloc_aligned( - mi_subproc_t* subproc, - size_t size, size_t alignment, size_t align_offset, - bool commit, bool allow_large, - mi_arena_id_t req_arena_id, mi_memid_t* memid) -{ - // if we cannot use OS allocation, return NULL - if (mi_option_is_enabled(mi_option_disallow_os_alloc) || req_arena_id != _mi_arena_id_none()) { - errno = ENOMEM; - return NULL; - } - - if (align_offset > 0) { - return _mi_os_alloc_aligned_at_offset(subproc, size, alignment, align_offset, commit, allow_large, memid); - } - else { - return _mi_os_alloc_aligned(subproc, size, alignment, commit, allow_large, memid); - } -} - - -// Allocate large sized memory -void* _mi_arenas_alloc_aligned( mi_heap_t* heap, - size_t size, size_t alignment, size_t align_offset, - bool commit, bool allow_large, - mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid) -{ - mi_assert_internal(memid != NULL); - mi_assert_internal(size > 0); - - // try to allocate in an arena if the alignment is small enough and the object is not too small (as for theap meta data) - if (!mi_option_is_enabled(mi_option_disallow_arena_alloc) && // is arena allocation allowed? - size >= MI_ARENA_MIN_OBJ_SIZE && size <= mi_arena_max_object_size() && // and not too small or too large - alignment <= MI_ARENA_SLICE_ALIGN && align_offset == 0) // and good alignment - { - const size_t slice_count = mi_slice_count_of_size(size); - void* p = mi_arenas_try_alloc(heap, slice_count, alignment, commit, allow_large, req_arena, tseq, numa_node, memid); - if (p != NULL) return p; - } - - // fall back to the OS - void* p = mi_arena_os_alloc_aligned(heap->subproc, size, alignment, align_offset, commit, allow_large, req_arena, memid); - return p; -} - -void* _mi_arenas_alloc(mi_heap_t* heap, size_t size, bool commit, bool allow_large, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid) -{ - return _mi_arenas_alloc_aligned(heap, size, MI_ARENA_SLICE_SIZE, 0, commit, allow_large, req_arena, tseq, numa_node, memid); -} - - - -/* ----------------------------------------------------------- - Arena page allocation ------------------------------------------------------------ */ - -// release ownership of a page. This may free the page if all blocks were concurrently -// freed in the meantime. Returns true if the page was freed. -static bool mi_abandoned_page_unown(mi_page_t* page, mi_theap_t* current_theapx) { - mi_assert_internal(mi_page_is_owned(page)); - mi_assert_internal(mi_page_is_abandoned(page)); - mi_assert_internal(mi_theap_matches_thread(current_theapx)); - mi_thread_free_t tf_new; - mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free); - do { - mi_assert_internal(mi_tf_is_owned(tf_old)); - while mi_unlikely(mi_tf_block(tf_old) != NULL) { - _mi_page_free_collect(page, false); // update used - if (mi_page_all_free(page)) { // it may become free just before unowning it - _mi_arenas_page_unabandon(page, current_theapx); - _mi_arenas_page_free(page, current_theapx); - return true; - } - tf_old = mi_atomic_load_relaxed(&page->xthread_free); - } - mi_assert_internal(mi_tf_block(tf_old)==NULL); - tf_new = mi_tf_create(NULL, false); - } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new)); - return false; -} - - -static bool mi_arena_try_claim_abandoned(size_t slice_index, mi_arena_t* arena, bool* keep_abandoned) { - // found an abandoned page of the right size - mi_page_t* const page = mi_arena_page_at_slice(arena, slice_index); - // can we claim ownership? - if (!mi_page_claim_ownership(page)) { - // there was a concurrent free that reclaims this page .. - // we need to keep it in the abandoned map as the free will call `mi_arena_page_unabandon`, - // and wait for readers (us!) to finish. This is why it is very important to set the abandoned - // bit again (or otherwise the unabandon will never stop waiting). - *keep_abandoned = true; - return false; - } - else { - // yes, we can reclaim it, keep the abandoned map entry clear - *keep_abandoned = false; - return true; - } -} - -// allocate initial arena_pages from the main heap -static mi_arena_pages_t* mi_arena_pages_alloc(mi_arena_t* arena); - -static mi_arena_pages_t* mi_heap_arena_pages(mi_heap_t* heap, mi_arena_t* arena) { - mi_assert_internal(arena!=NULL); - mi_assert_internal(heap!=NULL); - mi_assert(arena->arena_idx < MI_MAX_ARENAS); - return mi_atomic_load_ptr_acquire(mi_arena_pages_t, &heap->arena_pages[arena->arena_idx]); -} - -static mi_arena_t* mi_page_arena_pages(mi_page_t* page, size_t* slice_index, size_t* slice_count, mi_arena_pages_t** parena_pages) { - // todo: maybe store the arena* directly in the page? - mi_assert_internal(mi_page_is_owned(page)); - mi_arena_t* const arena = mi_arena_from_memid(page->memid, slice_index, slice_count); - mi_assert_internal(arena != NULL); - if (parena_pages != NULL) { - mi_heap_t* heap = mi_page_heap(page); - mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(heap, arena); - mi_assert_internal(arena_pages != NULL); - mi_assert_internal(slice_index==NULL || mi_bitmap_is_set(arena_pages->pages, *slice_index)); - *parena_pages = arena_pages; - } - return arena; -} - -static mi_arena_pages_t* mi_heap_ensure_arena_pages(mi_heap_t* heap, mi_arena_t* arena) { - mi_assert_internal(arena!=NULL); - mi_assert_internal(heap!=NULL); - mi_assert(arena->arena_idx < MI_MAX_ARENAS); - mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena); - if (arena_pages==NULL) { - mi_lock(&heap->arena_pages_lock) { - arena_pages = mi_atomic_load_ptr_acquire(mi_arena_pages_t, &heap->arena_pages[arena->arena_idx]); - if (arena_pages == NULL) { // still NULL? - if (_mi_is_heap_main(heap)) { - // the page info for the main heap is always allocated as part of an arena - arena_pages = &arena->pages_main; - } - else { - // always allocate the arena pages info from the main heap - // todo: allocate into the current arena? - arena_pages = mi_arena_pages_alloc(arena); - } - mi_atomic_store_ptr_release(mi_arena_pages_t, &heap->arena_pages[arena->arena_idx], arena_pages); - } - } - } - if (_mi_is_heap_main(heap)) { mi_assert(arena_pages != NULL); } // can never fail - return arena_pages; -} - -static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t slice_count, size_t block_size) -{ - mi_heap_t* const heap = _mi_theap_heap(theap); - const size_t tseq = theap->tld->thread_seq; - mi_arena_t* const req_arena = heap->exclusive_arena; - - MI_UNUSED(slice_count); - const size_t bin = _mi_bin(block_size); - if (bin >= MI_ARENA_BIN_COUNT) { - return NULL; // singleton page size - } - - // any abandoned in our size class? - mi_assert_internal(heap != NULL); - if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) == 0) { - return NULL; - } - - // search arena's - const bool allow_large = true; - const int any_numa = -1; - const bool match_numa = true; - mi_forall_suitable_arenas(heap, req_arena, tseq, match_numa, any_numa, allow_large, arena) - { - mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(heap, arena); - if (arena_pages != NULL) { - size_t slice_index; - mi_bitmap_t* const bitmap = arena_pages->pages_abandoned[bin]; - - if (mi_bitmap_try_find_and_claim(bitmap, tseq, &slice_index, &mi_arena_try_claim_abandoned, arena)) { - // found an abandoned page of the right size - // and claimed ownership. - mi_page_t* page = mi_arena_page_at_slice(arena, slice_index); - mi_assert_internal(mi_page_is_owned(page)); - mi_assert_internal(mi_page_is_abandoned(page)); - mi_assert_internal(mi_heap_has_page(heap, arena, page)); - mi_atomic_decrement_relaxed(&heap->abandoned_count[bin]); - mi_theap_stat_decrease(theap, pages_abandoned, 1); - mi_theap_stat_counter_increase(theap, pages_reclaim_on_alloc, 1); - - _mi_page_free_collect(page, false); // update `used` count - mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); - mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); - mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count)); - mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); - mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); - mi_assert_internal(mi_page_block_size(page) == block_size); - mi_assert_internal(!mi_page_is_full(page)); - return page; - } - } - } - mi_forall_suitable_arenas_end(); - return NULL; -} - -static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_count, size_t max_page_meta_count, size_t block_alignment, bool os_align, bool commit, mi_memid_t* memid, mi_arena_pages_t** parena_pages ) { - MI_UNUSED(max_page_meta_count); - mi_assert_internal(parena_pages!=NULL); - - *parena_pages = NULL; - const bool allow_large = (MI_SECURE < 5); // 5 = guard page at end of each arena page - const size_t page_alignment = MI_ARENA_SLICE_ALIGN; - - mi_heap_t* const heap = _mi_theap_heap(theap); - mi_tld_t* const tld = theap->tld; - mi_arena_t* const req_arena = heap->exclusive_arena; - const int numa_node = (heap->numa_node >= 0 ? heap->numa_node : tld->numa_node); - - // try to allocate from free space in arena's - uint8_t* start = NULL; - *memid = _mi_memid_none(); - const size_t alloc_size = mi_size_of_slices(slice_count); - if (!mi_option_is_enabled(mi_option_disallow_arena_alloc) && // allowed to allocate from arena's? - !os_align && // not large alignment - slice_count <= mi_arena_max_object_size()/MI_ARENA_SLICE_SIZE) // and not too large - { - start = (uint8_t*)mi_arenas_try_alloc(heap, slice_count, page_alignment, commit, allow_large, req_arena, tld->thread_seq, numa_node, memid); - if (start != NULL) { - mi_arena_pages_t* const arena_pages = mi_heap_ensure_arena_pages(heap, memid->mem.arena.arena); - *parena_pages = arena_pages; - if (arena_pages==NULL) { - _mi_arenas_free(heap->subproc, start, mi_size_of_slices(slice_count), *memid); // roll back - start = NULL; - } - else { - // note: the following assert should hold if we could check it atomically, but in a concurrent setting we may already allocate in slice_count - // mi_assert_internal(mi_bitmap_is_clearN(arena_pages->pages, memid->mem.arena.slice_index, memid->mem.arena.slice_count)); - mi_assert_internal(mi_bitmap_is_clear(arena_pages->pages, memid->mem.arena.slice_index)); - // don't set yet: mi_bitmap_set(arena_pages->pages, memid->mem.arena.slice_index); - } - } - } - - // otherwise fall back to the OS - if (start == NULL) { - #if MI_PAGE_META_IS_ALIGNED - size_t page_offset; // offset in the block for the page area - uint8_t* os_start; - if (block_alignment < MI_PAGE_META_ALIGNMENT) { - page_offset = (block_alignment < MI_PAGE_ALIGN ? MI_PAGE_ALIGN : block_alignment); - os_start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, MI_PAGE_META_ALIGNMENT, 0 /* align offset */, false /* commit */, false /* allow large */, req_arena, memid); - } - else { - // if we allow alignment >= MI_PAGE_META_ALIGNMENT we need to substract 1 from a pointer - // in _mi_aligned_ptr_page (and test for (intptr_t)p < 0 instead of NULL). We avoid this by limiting the max alignment. - _mi_warning_message("requested alignment is too large (%zu KiB)\n", block_alignment / MI_KiB); - errno = EINVAL; - return NULL; - // page_offset = MI_PAGE_META_ALIGNMENT; - // os_start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, block_alignment, MI_PAGE_META_ALIGNMENT /* align offset */, false /* commit */, false /* allow large */, req_arena, memid); - } - if (os_start==NULL) return NULL; - // commit page info and page - const size_t min_page_count = _mi_divide_up(page_offset + MI_ARENA_SLICE_SIZE,MI_ARENA_SLICE_SIZE) + max_page_meta_count; - const size_t min_page_meta = min_page_count * sizeof(mi_page_t); - mi_assert_internal(min_page_meta < page_offset); - bool is_zero; - bool ok = mi_arena_commit(heap->subproc,req_arena,os_start,min_page_meta,&is_zero,min_page_meta /* don't count in stats? */); - if (ok && commit) { - ok = mi_arena_commit(heap->subproc,req_arena,os_start + page_offset,alloc_size,NULL,0); - } - if (!ok) { _mi_os_free(heap->subproc,os_start,alloc_size+page_offset,*memid); return NULL; } - if (!is_zero && !memid->initially_zero) { - _mi_memzero_aligned(os_start,min_page_meta); - } - start = os_start + page_offset; - mi_assert_internal(_mi_is_aligned(start,block_alignment)); - mi_assert_internal(_mi_is_aligned(os_start,MI_PAGE_META_ALIGNMENT)); - mi_assert_internal(_mi_align_down_ptr(start-1,MI_PAGE_META_ALIGNMENT) == os_start); - mi_assert_internal((uint8_t*)_mi_aligned_ptr_page0(start) < os_start + min_page_meta); - memid->initially_committed = true; // so we don't commit again - #else - if (os_align) { - // note: slice_count already includes the page - start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size, block_alignment, page_alignment /* align offset */, commit, allow_large, req_arena, memid); - mi_assert_internal(_mi_is_aligned(start + page_alignment, block_alignment)); - } - else { - start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size, page_alignment, 0 /* align offset */, commit, allow_large, req_arena, memid); - } - #endif - if (start!=NULL) { mi_heap_stat_increase(heap,pages_os_allocated,1); } - } - - if (start == NULL) return NULL; - mi_assert_internal(_mi_is_aligned(start, MI_PAGE_ALIGN)); - return start; -} - -// Only used for non-separate pages -mi_decl_maybe_unused static size_t mi_page_block_start(size_t block_size, bool os_align) -{ - size_t offset; - #if MI_GUARDED - // in a guarded build, we align pages with blocks a multiple of an OS page size, to the OS page size - // this ensures that all blocks in such pages are OS page size aligned (which is needed for the guard pages) - const size_t os_page_size = _mi_os_page_size(); - mi_assert_internal(MI_PAGE_ALIGN >= os_page_size); - if (!os_align && block_size % os_page_size == 0 && block_size > os_page_size /* at least 2 or more */ ) { - offset = _mi_align_up(mi_page_info_size(), os_page_size); - } - else - #endif - if (os_align) { - offset = MI_PAGE_ALIGN; - } - else if (block_size != 0 && _mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) { - // naturally align power-of-2 blocks up to MI_PAGE_MAX_START_BLOCK_ALIGN2 size (4KiB) - offset = _mi_align_up(mi_page_info_size(), block_size); - if (block_size < 64) { offset += 3*block_size; } - } - else if (block_size != 0 && block_size <= MI_SMALL_SIZE_MAX) { - // align small blocks to their size - offset = _mi_align_up(mi_page_info_size(), block_size); - } - else if (block_size != 0 && (block_size % MI_PAGE_OSPAGE_BLOCK_ALIGN2) == 0) { - // also align large pages that are a multiple of MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4KiB) - offset = _mi_align_up(mi_page_info_size(), MI_PAGE_OSPAGE_BLOCK_ALIGN2); - } - else { - // otherwise start after the info - offset = mi_page_info_size(); - } - return _mi_align_up(offset,MI_MAX_ALIGN_SIZE); -} - - -// Free a page without modifying page_bin stats -static void mi_arenas_page_free_prim(mi_page_t* page); - -static mi_page_t* mi_arena_page_meta(mi_memid_t memid_slice, const void* slice_start) { - #if MI_PAGE_META_IS_ALIGNED - if (memid_slice.memkind == MI_MEM_ARENA || mi_memid_is_os(memid_slice)) { - // ensure the meta data is committed - if (memid_slice.memkind == MI_MEM_ARENA) { - mi_arena_t* const arena = memid_slice.mem.arena.arena; - uint8_t* const meta_slices = (uint8_t*)_mi_align_down_ptr(slice_start,MI_PAGE_META_ALIGNMENT); - mi_assert_internal(meta_slices >= mi_arena_start(arena)); - const size_t meta_slice_index = (meta_slices - mi_arena_start(arena)) / MI_ARENA_SLICE_SIZE; - if mi_unlikely(mi_bitmap_is_clear(arena->slices_committed, meta_slice_index)) { - // try to commit all page meta slices now - const size_t meta_slice_count = mi_arena_page_meta_aligned_slice_count(); - // the following assertion does not hold in a concurrent setting.. - // mi_assert_internal(mi_bitmap_is_clearN(arena->slices_committed, meta_slice_index, meta_slice_count)); - const size_t commit_size = meta_slice_count * MI_ARENA_SLICE_SIZE; - if (!mi_arena_commit(arena->subproc, arena, meta_slices, commit_size, NULL, 0)) { - // if the commit fails return NULL - return NULL; - } - // set the commit bits - mi_bitmap_setN(arena->slices_committed, meta_slice_index, meta_slice_count, NULL); - } - } - mi_page_t* const page_meta = _mi_aligned_ptr_page0(slice_start); - return page_meta; - } - #else - if (memid_slice.memkind == MI_MEM_ARENA) { - MI_UNUSED(slice_start); - mi_arena_t* const arena = memid_slice.mem.arena.arena; - if (arena->pages_meta != NULL) { - mi_assert_internal(MI_PAGE_META_IS_SEPARATED!=0); - return &arena->pages_meta[memid_slice.mem.arena.slice_index]; - } - } - #endif - return NULL; -} - -// Allocate a fresh page -static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool commit) -{ - const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN); - const bool singleton = (os_align || block_size > MI_LARGE_MAX_OBJ_SIZE); - const size_t max_page_meta_count = (singleton && slice_count > 2 ? 2 : slice_count); - - const size_t alloc_size = mi_size_of_slices(slice_count); - mi_memid_t memid = _mi_memid_none(); - mi_arena_pages_t* arena_pages = NULL; - uint8_t* const slice_start = mi_arenas_page_alloc_fresh_area(theap,slice_count,max_page_meta_count,block_alignment,os_align,commit,&memid,&arena_pages); - if (!slice_start) return NULL; - - // guard page at the end of mimalloc page? - #if MI_SECURE>=5 - mi_assert(alloc_size > _mi_os_secure_guard_page_size()); - const size_t page_noguard_size = alloc_size - _mi_os_secure_guard_page_size(); - #else - const size_t page_noguard_size = alloc_size; - #endif - - // allocate the page meta info - mi_page_t* page = NULL; - bool page_meta_is_separate = false; - size_t block_start = 0; - - // allocate page meta info at the arena start? - mi_page_t* const page_meta = mi_arena_page_meta(memid,slice_start); - if (page_meta!=NULL) { - mi_assert_internal(page_meta->block_size == 0); - #if MI_PAGE_META_SMALL_IS_ALIGNED - // if `block_size <= MI_SMALL_MAX_OBJ_SIZE` we put the page info in front of the slice, - // (note: it is important that `page_meta->block_size == 0` for `mi_arena_page_at_slice`) - if (!os_align && block_size <= MI_SMALL_MAX_OBJ_SIZE) { - // put page info in front of the slice - page = (mi_page_t*)slice_start; - block_start = mi_page_block_start(block_size, os_align); - } - else - #endif - { - page_meta_is_separate = true; - page = page_meta; - block_start = 0; - #if !defined(MI_PAGE_BLOCK_START_MAX_OFFSET) - #define MI_PAGE_BLOCK_START_MAX_OFFSET (8*MI_INTPTR_BITS) /* 512 */ - #endif - if (block_size >= MI_SIZE_SIZE && block_size <= MI_PAGE_BLOCK_START_MAX_OFFSET && - _mi_is_power_of_two(block_size)) - { - block_start = _mi_align_up(mi_page_info_size(), block_size); // to maintain natural alignment - if (block_size < 64) { block_start += 3*block_size; } - } - mi_assert_internal(page->block_size == 0); - _mi_memzero_aligned(page, sizeof(*page)); - } - } - if (page == NULL) { - #if MI_PAGE_META_IS_ALIGNED - // can only happen on failing to commit the page meta info - _mi_arenas_free(_mi_theap_subproc(theap),slice_start,alloc_size,memid); - return NULL; - #else - // put page meta info in front of the slice - page = (mi_page_t*)slice_start; - block_start = mi_page_block_start(block_size, os_align); - #endif - } - mi_assert_internal(block_size < MI_MAX_ALIGN_SIZE || block_start % MI_MAX_ALIGN_SIZE == 0); - if (_mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) { - mi_assert_internal(block_start % block_size == 0); // natural alignment (see also alloc_aligned.c) - } - - // commit first block? - size_t commit_size = 0; - if (!memid.initially_committed) { - commit_size = _mi_align_up(block_start + block_size, mi_page_min_commit_size()); - if (commit_size > page_noguard_size) { commit_size = page_noguard_size; } - bool is_zero = false; - if mi_unlikely(!mi_arena_commit( _mi_theap_subproc(theap), mi_memid_arena(memid), slice_start, commit_size, &is_zero, 0)) { - _mi_arenas_free(_mi_theap_subproc(theap), slice_start, alloc_size, memid); - return NULL; - } - } - // now we can finish initalization and use `mi_arenas_free_page_prim` on error - - // zero initialize the page meta data - if (!memid.initially_zero && !page_meta_is_separate) { - _mi_memzero_aligned(page, sizeof(*page)); - } - - // set the guard page - #if MI_SECURE>=5 - if (memid.initially_committed) { - _mi_os_secure_guard_page_set_at(_mi_theap_subproc(theap), slice_start + page_noguard_size, memid); - } - #endif - - // claimed free slices: initialize the page partly - if (!memid.initially_zero && memid.initially_committed) { - mi_track_mem_undefined(slice_start, slice_count * MI_ARENA_SLICE_SIZE); - } - else if (memid.initially_committed) { - mi_track_mem_defined(slice_start, slice_count * MI_ARENA_SLICE_SIZE); - } - #if MI_DEBUG > 1 - if (memid.initially_zero && memid.initially_committed) { - if (!mi_mem_is_zero(slice_start, page_noguard_size)) { - _mi_error_message(EFAULT, "internal error: page memory was not zero initialized.\n"); - memid.initially_zero = false; - if (block_start > 0) { _mi_memzero_aligned(page, sizeof(*page)); } - } - } - #endif - const size_t reserved = (os_align ? 1 : (page_noguard_size - block_start) / block_size); - mi_assert_internal(reserved > 0 && reserved <= UINT16_MAX); - - // initialize the page start - uint8_t* const start = slice_start + block_start; - mi_assert_internal(start > (uint8_t*)page); - page->page_offset = start - (uint8_t*)page; - - // initialize page meta-data - page->reserved = (uint16_t)reserved; - page->block_size = block_size; - page->memid = memid; - page->free_is_zero = memid.initially_zero; - - mi_assert_internal((commit && commit_size==0) || (!commit && (commit_size <= UINT16_MAX * _mi_os_page_size()))); - page->slice_pcommitted = (uint16_t)(commit_size / _mi_os_page_size()); - - page->heap = _mi_theap_heap(theap); - mi_page_set_theap(page,theap); - // mi_assert_internal(mi_page_theap(page) == _mi_heap_theap_peek(page->heap)) - - #if MI_PAGE_META_IS_ALIGNED - mi_assert_internal(page_meta!=NULL); - mi_atomic_store_ptr_release(mi_page_t,&page_meta->self,page); - if (slice_count > 1) { - // at least two for large singleton blocks as guard pages can have a large offset beyond a single slice - for(size_t i = 1; i < max_page_meta_count; i++) { - mi_assert_internal(page_meta[i].block_size == 0); - mi_atomic_store_ptr_release(mi_page_t,&page_meta[i].self,page); - } - } - #if MI_DEBUG>1 - mi_page_t* pstart = _mi_aligned_ptr_page0(slice_start); - mi_assert_internal(mi_atomic_load_ptr_acquire(mi_page_t,&pstart->self)==page); - if (reserved>1) { - mi_page_t* pend = _mi_aligned_ptr_page0(slice_start + (slice_count*MI_ARENA_SLICE_SIZE) - 1); - mi_assert_internal(mi_atomic_load_ptr_acquire(mi_page_t,&pend->self)==page); - } - #endif - #endif - - mi_assert_internal(page->free==NULL); - mi_assert_internal(page_meta_is_separate == mi_page_meta_is_separated(page)); - mi_assert_internal(mi_page_slice_start(page) == slice_start); - mi_assert_internal(mi_page_size(page) <= page_noguard_size); - - - // now register in the arena_pages - if (arena_pages!=NULL) { - mi_assert_internal(memid.memkind == MI_MEM_ARENA); - mi_bitmap_set(arena_pages->pages, memid.mem.arena.slice_index); - } - - // and own it - mi_page_claim_ownership(page); - - // register in the page map - if mi_unlikely(!_mi_page_map_register(page)) { - mi_arenas_page_free_prim(page); - return NULL; - } - - // stats - mi_theap_stat_increase(theap, pages, 1); - mi_theap_stat_increase(theap, page_bins[_mi_page_stats_bin(page)], 1); - - mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page),MI_PAGE_ALIGN)); - mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); - mi_assert_internal(mi_page_block_size(page) == block_size); - // mi_assert_internal(mi_page_is_abandoned(page)); - mi_assert_internal(mi_page_is_owned(page)); - - return page; -} - -// Allocate a regular small/medium/large page. -static mi_page_t* mi_arenas_page_regular_alloc(mi_theap_t* theap, size_t slice_count, size_t block_size) -{ - // 1. look for an abandoned page - mi_page_t* page = mi_arenas_page_try_find_abandoned(theap, slice_count, block_size); - if (page != NULL) { - return page; // return as abandoned - } - - // 2. find a free block, potentially allocating a new arena - const long commit_on_demand = mi_option_get(mi_option_page_commit_on_demand); - const bool commit = (mi_page_min_commit_size() >= slice_count * MI_ARENA_SLICE_SIZE || // always commit small pages - (slice_count >= mi_slice_count_of_size(UINT16_MAX * _mi_os_page_size())) || // always commit pages too large to hold a 32-bit slice_committed - (commit_on_demand == 2 && _mi_os_has_overcommit()) || (commit_on_demand == 0)); - page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, 1, commit); - if (page == NULL) return NULL; - - mi_assert_internal(page->memid.memkind != MI_MEM_ARENA || page->memid.mem.arena.slice_count == slice_count); - if (!_mi_page_init(theap, page)) { - _mi_arenas_page_free(page,theap); - return NULL; - } - - return page; -} - -// Allocate a page containing one block (very large, or with large alignment) -static mi_page_t* mi_arenas_page_singleton_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment) -{ - #if MI_PAGE_META_IS_ALIGNED - const size_t info_size = 0; - #else - const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN); - const size_t info_size = (os_align ? MI_PAGE_ALIGN : mi_page_info_size()); - #endif - #if MI_SECURE < 2 - const size_t slice_count = mi_slice_count_of_size(info_size + block_size); - #else - const size_t slice_count = mi_slice_count_of_size(_mi_align_up(info_size + block_size, _mi_os_secure_guard_page_size()) + _mi_os_secure_guard_page_size()); - #endif - - mi_page_t* page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, block_alignment, true /* commit singletons always */); - if (page == NULL) return NULL; - - mi_assert(page->reserved == 1); - if (!_mi_page_init(theap, page)) { - _mi_arenas_page_free(page,theap); - return NULL; - } - - return page; -} - - -mi_page_t* _mi_arenas_page_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment) { - mi_page_t* page; - // semi static assert: ensure that all non-singleton block size bins are covered. - mi_assert(_mi_bin(MI_LARGE_MAX_OBJ_SIZE) < MI_ARENA_BIN_COUNT); - if mi_unlikely(block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) { - mi_assert_internal(_mi_is_power_of_two(block_alignment)); - page = mi_arenas_page_singleton_alloc(theap, block_size, block_alignment); - } - else if (block_size <= MI_SMALL_MAX_OBJ_SIZE) { - page = mi_arenas_page_regular_alloc(theap, mi_slice_count_of_size(MI_SMALL_PAGE_SIZE), block_size); - } - else if (block_size <= MI_MEDIUM_MAX_OBJ_SIZE) { - page = mi_arenas_page_regular_alloc(theap, mi_slice_count_of_size(MI_MEDIUM_PAGE_SIZE), block_size); - } - #if MI_ENABLE_LARGE_PAGES - else if (block_size <= MI_LARGE_MAX_OBJ_SIZE) { - page = mi_arenas_page_regular_alloc(theap, mi_slice_count_of_size(MI_LARGE_PAGE_SIZE), block_size); - } - #endif - else { - page = mi_arenas_page_singleton_alloc(theap, block_size, block_alignment); - } - if mi_unlikely(page == NULL) { - return NULL; - } - // mi_assert_internal(page == NULL || _mi_page_segment(page)->subproc == tld->subproc); - mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); - mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); - mi_assert_internal(block_alignment <= MI_PAGE_MAX_OVERALLOC_ALIGN || _mi_is_aligned(mi_page_start(page), block_alignment)); - - return page; -} - -static void mi_arenas_page_free_prim(mi_page_t* page) { - mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); - mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); - mi_assert_internal(mi_page_is_owned(page)); - mi_assert_internal(mi_page_all_free(page)); - mi_assert_internal(page->next==NULL && page->prev==NULL); - - #if MI_DEBUG>1 - if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) { - size_t bin = _mi_bin(mi_page_block_size(page)); - size_t slice_index; - size_t slice_count; - mi_arena_pages_t* arena_pages = NULL; - mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); - mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); - mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); - mi_assert_internal(bin >= MI_ARENA_BIN_COUNT || mi_bitmap_is_clearN(arena_pages->pages_abandoned[bin], slice_index, 1)); - mi_assert_internal(mi_bitmap_is_setN(arena_pages->pages, slice_index, 1)); - // note: we cannot check for `!mi_page_is_abandoned_and_mapped` since that may - // be (temporarily) not true if the free happens while trying to reclaim - // see `mi_arena_try_claim_abandoned` - } - #endif - - // unregister page - _mi_page_map_unregister(page); - - // recommit guard page at the end? - // we must do this since we may later allocate large spans over this page and cannot have a guard page in between - #if MI_SECURE >= 5 - if (!page->memid.is_pinned) { - _mi_os_secure_guard_page_reset_before(mi_page_subproc(page), mi_page_slice_start(page) + mi_page_full_size(page), page->memid); - } - #endif - - // and free - if (page->memid.memkind == MI_MEM_ARENA) { - mi_arena_pages_t* arena_pages; - size_t slice_index; - size_t slice_count; MI_UNUSED(slice_count); - mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); - mi_assert_internal(arena_pages!=NULL); - mi_assert_internal(arena->subproc == mi_page_subproc(page)); - mi_bitmap_clear(arena_pages->pages, slice_index); - const size_t slice_committed = mi_page_slice_committed(page); - if (slice_committed > 0) { - // if committed on-demand, set the commit bits to account commit properly - mi_assert_internal(mi_page_full_size(page) >= slice_committed); - const size_t total_slices = slice_committed / MI_ARENA_SLICE_SIZE; // conservative - //mi_assert_internal(mi_bitmap_is_clearN(arena->slices_committed, slice_index, total_slices)); - mi_assert_internal(slice_count >= total_slices); - if (total_slices > 0) { - mi_bitmap_setN(arena->slices_committed, slice_index, total_slices, NULL); - } - // any left over? - const size_t extra = slice_committed % MI_ARENA_SLICE_SIZE; - if (extra > 0) { - // pretend it was decommitted already - mi_subproc_stat_decrease(arena->subproc, committed, extra); - } - } - else { - mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); - } - } - else { - mi_heap_stat_decrease(page->heap, pages_os_allocated, 1); - } - if (mi_page_meta_is_separated(page)) { page->block_size = 0; } // for assertion checking - _mi_arenas_free( mi_page_subproc(page), mi_page_slice_start(page), mi_page_full_size(page), page->memid); -} - -void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) { - mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); - mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); - mi_assert_internal(mi_page_is_owned(page)); - mi_assert_internal(mi_page_all_free(page)); - mi_assert_internal(mi_page_is_abandoned(page)); - mi_assert_internal(page->next==NULL && page->prev==NULL); - mi_assert_internal(mi_theap_matches_thread(current_theapx)); - - mi_heap_t* const heap = mi_page_heap(page); - mi_theapx_stat_decrease(heap, current_theapx, page_bins[_mi_page_stats_bin(page)], 1); - mi_theapx_stat_decrease(heap, current_theapx, pages, 1); - _mi_page_free_collect(page,false); // update used count for cross-thread free's - _mi_page_update_stats(page); // and update the stats - mi_arenas_page_free_prim(page); -} - -/* ----------------------------------------------------------- - Arena abandon ------------------------------------------------------------ */ - -void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theapx) { - mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); - mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); - mi_assert_internal(mi_page_is_owned(page)); - mi_assert_internal(mi_page_is_abandoned(page)); - mi_assert_internal(!mi_page_all_free(page)); - mi_assert_internal(page->next==NULL && page->prev == NULL); - mi_assert_internal(mi_theap_matches_thread(current_theapx)); - // mi_assert_internal(current_theap == _mi_page_associated_theap(page)); - - // note: somewhat expensive to update here, but might be good as then we attribute - // the current allocations/frees to the current thread/theap. Otherwise it might be - // reclaimed later in another thread/theap and those allocations/frees get attributed there... - _mi_page_update_stats(page); - - // add to abandoned? - mi_heap_t* heap = mi_page_heap(page); - if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) { - // make available for allocations - size_t bin = _mi_bin(mi_page_block_size(page)); - mi_assert_internal(bin < MI_ARENA_BIN_COUNT); - if (bin < MI_ARENA_BIN_COUNT) { // paranoia - size_t slice_index; - size_t slice_count; - mi_arena_pages_t* arena_pages = NULL; - mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); MI_UNUSED(arena); - - mi_assert_internal(!mi_page_is_singleton(page)); - mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); - mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); - mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count)); - - mi_page_set_abandoned_mapped(page); - const bool was_clear = mi_bitmap_set(arena_pages->pages_abandoned[bin], slice_index); - MI_UNUSED(was_clear); mi_assert_internal(was_clear); - mi_atomic_increment_relaxed(&heap->abandoned_count[bin]); - mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1); - mi_abandoned_page_unown(page, current_theapx); - return; - } - } - // otherwise, - // page is full (or a singleton), or the page is OS/externally allocated - // leave as is; it will be reclaimed when an object is free'd in the page - // but for non-arena pages, add to the subproc list so these can be visited - if (page->memid.memkind != MI_MEM_ARENA) { - mi_lock(&heap->os_abandoned_pages_lock) { - // push in front - page->prev = NULL; - page->next = heap->os_abandoned_pages; - if (page->next != NULL) { page->next->prev = page; } - heap->os_abandoned_pages = page; - } - mi_theapx_stat_increase(heap, current_theapx, pages_os_abandoned, 1); - } - mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1); - mi_abandoned_page_unown(page, current_theapx); -} - - -// this is called from `free.c:mi_free_try_collect_mt` only. -bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page) { - mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); - mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); - mi_assert_internal(mi_page_is_owned(page)); - mi_assert_internal(mi_page_is_abandoned(page)); - mi_assert_internal(!mi_page_is_abandoned_mapped(page)); - mi_assert_internal(!mi_page_is_full(page)); - mi_assert_internal(!mi_page_all_free(page)); - mi_assert_internal(!mi_page_is_singleton(page)); - if (mi_page_is_full(page) || mi_page_is_abandoned_mapped(page) || page->memid.memkind != MI_MEM_ARENA) { - return false; - } - else { - // do not use _mi_heap_theap as we may call this during shutdown of threads and don't want to reinitialize the theap - mi_theap_t* const theapx = _mi_page_associated_theap_peek(page); // can be NULL - mi_heap_t* const heap = mi_page_heap(page); - // if (theapx==NULL) return false; - mi_theapx_stat_counter_increase(heap, theapx, pages_reabandon_full, 1); - mi_theapx_stat_adjust_decrease(heap, theapx, pages_abandoned, 1); // adjust as we are not abandoning fresh - _mi_arenas_page_abandon(page, theapx); - return true; - } -} - -// called from `mi_free` if trying to unabandon an abandoned page -void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) { - mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); - mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); - mi_assert_internal(mi_page_is_owned(page)); - mi_assert_internal(mi_page_is_abandoned(page)); - mi_assert_internal(mi_theap_matches_thread(current_theapx)); - - mi_heap_t* const heap = mi_page_heap(page); - if (mi_page_is_abandoned_mapped(page)) { - mi_assert_internal(page->memid.memkind==MI_MEM_ARENA); - // remove from the abandoned map - const size_t bin = _mi_bin(mi_page_block_size(page)); - mi_assert_internal(bin < MI_ARENA_BIN_COUNT); - size_t slice_index; - size_t slice_count; - mi_arena_pages_t* arena_pages; - mi_arena_t* arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); MI_UNUSED(arena); - - mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); - mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); - - // this busy waits until a concurrent reader (from alloc_abandoned) is done - mi_bitmap_clear_once_set(arena->subproc, arena_pages->pages_abandoned[bin], slice_index); - mi_page_clear_abandoned_mapped(page); - mi_atomic_decrement_relaxed(&heap->abandoned_count[bin]); - } - else { - // page is full (or a singleton), page is OS allocated - // if not an arena page, remove from the subproc os pages list - if (page->memid.memkind != MI_MEM_ARENA) { - mi_lock(&heap->os_abandoned_pages_lock) { - if (page->prev != NULL) { page->prev->next = page->next; } - if (page->next != NULL) { page->next->prev = page->prev; } - if (heap->os_abandoned_pages == page) { heap->os_abandoned_pages = page->next; } - page->next = NULL; - page->prev = NULL; - } - mi_theapx_stat_decrease(heap, current_theapx, pages_os_abandoned, 1); - } - } - mi_theapx_stat_decrease(heap, current_theapx, pages_abandoned, 1); -} - - -/* ----------------------------------------------------------- - Arena free ------------------------------------------------------------ */ -static void mi_arena_schedule_purge(mi_arena_t* arena, size_t slice_index, size_t slices); -static void mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq); - -void _mi_arenas_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid) { - if (p==NULL) return; - if (size==0) return; - - // need to set all memory to undefined as some parts may still be marked as no_access (like padding etc.) - mi_track_mem_undefined(p, size); - - if (mi_memkind_is_os(memid.memkind)) { - // was a direct OS allocation, pass through - _mi_os_free(subproc, p, size, memid); - } - else if (memid.memkind == MI_MEM_ARENA) { - // allocated in an arena - size_t slice_count; - size_t slice_index; - mi_arena_t* arena = mi_arena_from_memid(memid, &slice_index, &slice_count); - mi_assert_internal(arena!=NULL); - mi_assert_internal(arena->subproc == subproc); - mi_assert_internal((size%MI_ARENA_SLICE_SIZE)==0); - mi_assert_internal((slice_count*MI_ARENA_SLICE_SIZE)==size); - mi_assert_internal(mi_arena_slice_start(arena,slice_index) <= (uint8_t*)p); - mi_assert_internal(mi_arena_slice_start(arena,slice_index) + mi_size_of_slices(slice_count) > (uint8_t*)p); - // checks - if (arena == NULL) { - _mi_error_message(EINVAL, "trying to free from an invalid arena: %p, size %zu, memkind: 0x%x\n", p, size, memid.memkind); - return; - } - mi_assert_internal(slice_index < arena->slice_count); - mi_assert_internal(slice_index >= mi_arena_info_slices(arena)); - if (slice_index < mi_arena_info_slices(arena) || slice_index >= arena->slice_count) { - _mi_error_message(EINVAL, "trying to free from an invalid arena block: %p, size %zu, memkind: 0x%x\n", p, size, memid.memkind); - return; - } - - // potentially decommit - if (!arena->memid.is_pinned /* && !arena->memid.initially_committed */) { // todo: allow decommit even if initially committed? - // (delay) purge the page - mi_arena_schedule_purge(arena, slice_index, slice_count); - } - - // and make it available to others again - bool all_inuse = mi_bbitmap_setN(arena->slices_free, slice_index, slice_count); - if (!all_inuse) { - _mi_error_message(EAGAIN, "trying to free an already freed arena block: %p, size %zu\n", mi_arena_slice_start(arena,slice_index), mi_size_of_slices(slice_count)); - return; - }; - } - else if (memid.memkind == MI_MEM_MALLOC) { - _mi_free_subproc_safe(p); - } - else { - // arena was none, external, or static; nothing to do - mi_assert_internal(mi_memid_needs_no_free(memid)); - } - - // try to purge expired decommits - // mi_arenas_try_purge(false, false, NULL); -} - -// Purge the arenas; if `force_purge` is true, amenable parts are purged even if not yet expired -void _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld) { - mi_arenas_try_purge(force_purge, visit_all, tld->subproc, tld->thread_seq); -} - - -// Is a pointer contained in the given arena area? -static bool mi_arena_strictly_contains(mi_arena_t* arena, const void* p) { - return (arena != NULL && - mi_arena_start(arena) <= (const uint8_t*)p && - mi_arena_start(arena) + mi_size_of_slices(arena->slice_count) >(const uint8_t*)p); -} - -// Is a pointer inside any of our arenas? -static bool mi_arenas_contain_ex(const void* p, mi_arena_t* parent) { - mi_subproc_t* subproc = _mi_subproc(); - const size_t max_arena = mi_arenas_get_count(subproc); - for (size_t i = 0; i < max_arena; i++) { - mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]); - if (arena != NULL) { - if (parent==NULL || arena==parent || arena->parent==parent) { - if (mi_arena_strictly_contains(arena, p)) { - return true; - } - } - } - } - return false; -} - -// // Is a pointer inside any of our arenas? -// bool _mi_arenas_contain(const void* p) { -// return mi_arenas_contain_ex(p, NULL); -// } - -// Is a pointer contained in the given arena area? -bool mi_arena_contains(mi_arena_id_t arena_id, const void* p) { - mi_arena_t* arena = _mi_arena_from_id(arena_id); - if (arena==NULL) return false; - else if (mi_arena_strictly_contains(arena, p)) return true; - else return mi_arenas_contain_ex(p, arena); // maybe a subarena? -} - - -/* ----------------------------------------------------------- - Remove an arena. ------------------------------------------------------------ */ - -// destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit` -// for dynamic libraries that are unloaded and need to release all their allocated memory. -static void mi_arenas_unsafe_destroy(mi_subproc_t* subproc) { - mi_assert_internal(subproc != NULL); - const size_t arena_count = mi_arenas_get_count(subproc); - for (size_t i = 0; i < arena_count; i++) { - mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]); - if (arena != NULL) { - // mi_lock_done(&arena->abandoned_visit_lock); - mi_atomic_store_ptr_release(mi_arena_t, &subproc->arenas[i], NULL); - if (mi_memkind_is_os(arena->memid.memkind)) { - _mi_os_free_ex(subproc, mi_arena_start(arena), mi_arena_size(arena), true, arena->memid); - } - } - } - // try to lower the max arena. - size_t expected = arena_count; - mi_atomic_cas_strong_acq_rel(&subproc->arena_count, &expected, (size_t)0); -} - - -// destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit` -// for dynamic libraries that are unloaded and need to release all their allocated memory. -void _mi_arenas_unsafe_destroy_all(mi_subproc_t* subproc) { - mi_arenas_unsafe_destroy(subproc); - // mi_arenas_try_purge(true /* force purge */, true /* visit all*/, subproc, 0 /* thread seq */); // purge non-owned arenas -} - - -/* ----------------------------------------------------------- - Add an arena. ------------------------------------------------------------ */ - -static bool mi_arenas_add(mi_subproc_t* subproc, mi_arena_t* arena, mi_arena_id_t* arena_id) -{ - mi_assert_internal(arena != NULL); - mi_assert_internal(arena->slice_count > 0); - if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); } - - // try to find a NULL entry - mi_arena_t* expected; - size_t count = mi_arenas_get_count(subproc); - for( size_t i = 0; i < count; i++) { - if (mi_arena_from_index(subproc,i) == NULL) { - arena->arena_idx = i; - expected = NULL; - if (mi_atomic_cas_ptr_strong_release(mi_arena_t, &subproc->arenas[i], &expected, arena)) { - // success - if (arena_id != NULL) { *arena_id = mi_arena_id_from_arena(arena); } - return true; - } - } - } - - // otherwise, try to allocate a fresh slot - while(countarena_count, &count, count+1)) { - arena->arena_idx = count; - expected = NULL; - if (mi_atomic_cas_ptr_strong_release(mi_arena_t, &subproc->arenas[count], &expected, arena)) { - mi_subproc_stat_counter_increase(arena->subproc, arena_count, 1); - if (arena_id != NULL) { *arena_id = mi_arena_id_from_arena(arena); } - return true; - } - } - } - - // failed - arena->arena_idx = 0; - arena->subproc = NULL; - return false; -} - -static size_t mi_arena_pages_size(size_t slice_count, size_t* bitmap_base) { - if (slice_count == 0) slice_count = MI_BCHUNK_BITS; - mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0); - const size_t base_size = _mi_align_up(sizeof(mi_arena_pages_t), MI_BCHUNK_SIZE); - const size_t bitmaps_count = 1 + MI_ARENA_BIN_COUNT; // pages, and abandoned - const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL); - const size_t size = base_size + bitmaps_size; - if (bitmap_base != NULL) *bitmap_base = base_size; - return size; -} - -static mi_arena_t* mi_arena_info(void* area) { - return (mi_arena_t*)((uint8_t*)area + mi_size_of_slices(mi_arena_page_meta_aligned_slice_count())); -} - -static size_t mi_arena_info_slices_needed(size_t slice_count, size_t* bitmap_base) { - if (slice_count == 0) slice_count = MI_BCHUNK_BITS; - mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0); - const size_t base_size = mi_size_of_slices(mi_arena_page_meta_aligned_slice_count()) + _mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE); - const size_t bitmaps_count = 4 + MI_ARENA_BIN_COUNT; // commit, dirty, purge, pages, and abandoned - const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL) + mi_bbitmap_size(slice_count, NULL); // + free - #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_IS_ALIGNED - const size_t pages_size = slice_count * sizeof(mi_page_t); - #else - const size_t pages_size = 0; - #endif - const size_t size = base_size + bitmaps_size + pages_size; - - const size_t os_page_size = _mi_os_page_size(); - const size_t info_size = _mi_align_up(size, os_page_size) + _mi_os_secure_guard_page_size(); - const size_t info_slices = mi_slice_count_of_size(info_size); - - if (bitmap_base != NULL) *bitmap_base = base_size; - return info_slices; -} - -static mi_bitmap_t* mi_arena_bitmap_init(size_t slice_count, uint8_t** base) { - mi_bitmap_t* bitmap = (mi_bitmap_t*)(*base); - *base = (*base) + mi_bitmap_init(bitmap, slice_count, true /* already zero */); - return bitmap; -} - -static mi_bbitmap_t* mi_arena_bbitmap_init(mi_subproc_t* subproc, size_t slice_count, uint8_t** base) { - mi_bbitmap_t* bbitmap = (mi_bbitmap_t*)(*base); - *base = (*base) + mi_bbitmap_init(subproc, bbitmap, slice_count, true /* already zero */); - return bbitmap; -} - -static mi_arena_pages_t* mi_arena_pages_alloc(mi_arena_t* arena) { - const size_t slice_count = arena->slice_count; - size_t bitmap_base = 0; - const size_t size = mi_arena_pages_size(slice_count, &bitmap_base); - mi_arena_pages_t* arena_pages = (mi_arena_pages_t*)mi_heap_zalloc_aligned(arena->subproc->heap_main, size, MI_BCHUNK_SIZE); - if (arena_pages==NULL) return NULL; - uint8_t* base = (uint8_t*)arena_pages + bitmap_base; - mi_assert_internal(_mi_is_aligned(base, MI_BCHUNK_SIZE)); - arena_pages->pages = mi_arena_bitmap_init(slice_count, &base); - for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) { - arena_pages->pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base); - } - return arena_pages; -} - -static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, - size_t slice_count, mi_arena_t* parent, size_t total_size, - int numa_node, bool exclusive, - mi_memid_t memid, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) -{ - mi_assert_internal(_mi_is_aligned(start,MI_ARENA_ALIGNMENT)); - mi_assert_internal(mi_size_of_slices(slice_count)>=MI_ARENA_MIN_SIZE); - - if (slice_count > MI_BITMAP_MAX_BIT_COUNT) { // 16 GiB for now - // note: this should never happen if called from `mi_manage_os_memory` (as that allocates sub-arenas when needed) - _mi_warning_message("cannot use OS memory since it is too large (size %zu MiB, maximum is %zu MiB)", mi_size_of_slices(slice_count)/MI_MiB, mi_size_of_slices(MI_BITMAP_MAX_BIT_COUNT)/MI_MiB); - return NULL; - } - - size_t bitmap_base; - const size_t info_slices = mi_arena_info_slices_needed(slice_count, &bitmap_base); - if (slice_count < info_slices+1) { - _mi_warning_message("cannot use OS memory since it is not large enough (size %zu KiB, minimum required is %zu KiB)", mi_size_of_slices(slice_count)/MI_KiB, mi_size_of_slices(info_slices+1)/MI_KiB); - return NULL; - } - // else if (info_slices >= MI_ARENA_MAX_CHUNK_OBJ_SLICES) { - // _mi_warning_message("cannot use OS memory since it is too large with respect to the maximum object size (size %zu MiB, meta-info slices %zu, maximum object slices are %zu)", mi_size_of_slices(slice_count)/MI_MiB, info_slices, MI_ARENA_MAX_CHUNK_OBJ_SLICES); - // return NULL; - // } - - // commit & zero if needed - if (!memid.initially_committed) { - size_t commit_size = mi_size_of_slices(info_slices); - // leave a guard OS page decommitted at the end? - if (!memid.is_pinned) { commit_size -= _mi_os_secure_guard_page_size(); } - bool ok = false; - if (commit_fun != NULL) { - ok = (*commit_fun)(true /* commit */, start, commit_size, NULL, commit_fun_arg); - } - else { - ok = _mi_os_commit(subproc, start, commit_size, NULL); - } - if (!ok) { - _mi_warning_message("unable to commit meta-data for OS memory"); - return NULL; - } - } - else if (!memid.is_pinned) { - // if MI_SECURE, set a guard page at the end of the arena info - // todo: this does not respect the commit_fun as the memid is of external memory - _mi_os_secure_guard_page_set_before(subproc, (uint8_t*)start + mi_size_of_slices(info_slices), memid); - } - if (!memid.initially_zero) { - _mi_memzero(start, mi_size_of_slices(info_slices) - _mi_os_secure_guard_page_size()); - } - - // init - mi_arena_t* arena = mi_arena_info(start); - arena->start = start; - arena->subproc = subproc; - arena->memid = memid; - arena->is_exclusive = exclusive; - arena->slice_count = slice_count; - arena->info_slices = info_slices; - if (numa_node<0 && mi_option_is_enabled(mi_option_arena_is_numa_local)) { - arena->numa_node = _mi_os_numa_node(); - } - else { - arena->numa_node = numa_node; - } - arena->purge_expire = 0; - arena->commit_fun = commit_fun; - arena->commit_fun_arg = commit_fun_arg; - arena->parent = parent; - arena->total_size = total_size; - - // init bitmaps - uint8_t* base = mi_arena_start(arena) + bitmap_base; - arena->slices_free = mi_arena_bbitmap_init(subproc, slice_count, &base); - arena->slices_committed = mi_arena_bitmap_init(slice_count, &base); - arena->slices_dirty = mi_arena_bitmap_init(slice_count, &base); - arena->slices_purge = mi_arena_bitmap_init(slice_count, &base); - arena->pages_main.pages = mi_arena_bitmap_init(slice_count, &base); - for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) { - arena->pages_main.pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base); - } - #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_IS_ALIGNED - arena->pages_meta = (mi_page_t*)base; - base += (slice_count * sizeof(mi_page_t)); - #else - arena->pages_meta = NULL; - #endif - mi_assert_internal(mi_size_of_slices(info_slices) >= (size_t)(base - mi_arena_start(arena))); - - // reserve our meta info (and reserve slices outside the memory area) - #if MI_PAGE_META_IS_ALIGNED - for(size_t i = 0; i < arena->slice_count; i += MI_PAGE_META_ALIGNED_COUNT) { - // set all free slices (and skip the slices reserved for the page meta info) - const size_t meta_slices = (i==0 ? info_slices : mi_arena_page_meta_aligned_slice_count()); - const size_t start_idx = (i + meta_slices); - size_t count = MI_PAGE_META_ALIGNED_COUNT - meta_slices; - if (start_idx < arena->slice_count) { - if (count + start_idx > arena->slice_count) { - count = arena->slice_count - start_idx; - mi_assert_internal(count > 0); - } - mi_bbitmap_unsafe_setN(arena->slices_free, start_idx, count); - } - } - #else - mi_bbitmap_unsafe_setN(arena->slices_free, info_slices /* start */, arena->slice_count - info_slices); - #endif - if (memid.initially_committed) { - mi_bitmap_unsafe_setN(arena->slices_committed, 0, arena->slice_count); - } - if (!memid.initially_zero) { - mi_bitmap_unsafe_setN(arena->slices_dirty, 0, arena->slice_count); - } - - if (!mi_arenas_add(subproc, arena, arena_id)) { return NULL; } - return arena; -} - -static bool mi_manage_os_memory_ex2(mi_subproc_t* subproc, void* start, size_t size, int numa_node, bool exclusive, - mi_memid_t memid, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) mi_attr_noexcept -{ - // checks - mi_assert(start!=NULL); - if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); } - if (start==NULL) return false; - if (!_mi_is_aligned(start, MI_ARENA_ALIGNMENT)) { - // we can align the start since the memid tracks the real base of the memory. - void* const aligned_start = _mi_align_up_ptr(start, MI_ARENA_ALIGNMENT); - const size_t diff = (uint8_t*)aligned_start - (uint8_t*)start; - if (diff >= size || (size - diff) < MI_ARENA_ALIGNMENT) { - _mi_warning_message("after alignment, the size of the arena becomes too small (memory at %p with size %zu)\n", start, size); - return false; - } - start = aligned_start; - size = size - diff; - } - - // allocate enough arena's to span the full memory area - // the first arena is the owner, the rest are "sub-arena" (with `parent` pointing to the first one) - size_t total_slice_count = _mi_align_down(size / MI_ARENA_SLICE_SIZE, MI_BCHUNK_BITS); - size_t total_size = mi_size_of_slices(total_slice_count); - if (total_size < MI_ARENA_MIN_SIZE) { - _mi_warning_message("cannot use OS memory since it is not large enough (size %zu KiB, minimum required is %zu KiB)", size/MI_KiB, MI_ARENA_MIN_SIZE/MI_KiB); - return false; - } - - mi_arena_t* parent = NULL; - do { - // counting down on the total_slice_count - size_t slice_count = total_slice_count; - if (slice_count > MI_BITMAP_MAX_BIT_COUNT) { // 16 GiB for now (with 64KiB slices) - slice_count = MI_BITMAP_MAX_BIT_COUNT; - } - - // initialize - mi_arena_t* arena = mi_arena_initialize( subproc, start, slice_count, parent, - (parent==NULL ? total_size : 0), numa_node, exclusive, - memid, commit_fun, commit_fun_arg, - (parent==NULL ? arena_id : NULL)); - if (arena==NULL) { - // failed to initialize due to failing commit or too many arena's - if (parent==NULL) { - return false; - } - else { - // partial success, but failed to use the full area.. - // todo: roll-back in this case? that requires a lock on the arena's array though - mi_assert(mi_size_of_slices(total_slice_count) <= parent->total_size); - parent->total_size -= mi_size_of_slices(total_slice_count); - return true; - } - } - - // success - if (parent==NULL) { - parent = arena; - memid.memkind = MI_MEM_NONE; - } - mi_assert(slice_count <= total_slice_count); - total_slice_count -= slice_count; - start = (uint8_t*)start + mi_size_of_slices(slice_count); - } - while (total_slice_count > 0); - - return true; -} - -bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept { - mi_memid_t memid = _mi_memid_create(MI_MEM_EXTERNAL); - memid.mem.os.base = start; - memid.mem.os.size = size; - memid.initially_committed = is_committed; - memid.initially_zero = is_zero; - memid.is_pinned = is_pinned; - return mi_manage_os_memory_ex2(_mi_subproc(), start, size, numa_node, exclusive, memid, NULL, NULL, arena_id); -} - -bool mi_manage_memory(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) mi_attr_noexcept -{ - mi_memid_t memid = _mi_memid_create(MI_MEM_EXTERNAL); - memid.mem.os.base = start; - memid.mem.os.size = size; - memid.initially_committed = is_committed; - memid.initially_zero = is_zero; - memid.is_pinned = is_pinned; - return mi_manage_os_memory_ex2(_mi_subproc(), start, size, numa_node, exclusive, memid, commit_fun, commit_fun_arg, arena_id); -} - - -// Reserve a range of regular OS memory -static int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) { - if (arena_id != NULL) *arena_id = _mi_arena_id_none(); - if (size <= MI_MAX_ALLOC_SIZE) { - size = _mi_align_up(size, MI_ARENA_SLICE_SIZE); // at least one slice - } - if (size > MI_MAX_ALLOC_SIZE) { - _mi_error_message(EOVERFLOW, "memory reservation request is too large (size %zu)\n", size); - return ENOMEM; - } - mi_memid_t memid; - void* start = _mi_os_alloc_aligned(subproc, size, MI_ARENA_ALIGNMENT, commit, allow_large, &memid); - if (start == NULL) return ENOMEM; - if (!mi_manage_os_memory_ex2(subproc, start, size, -1 /* numa node */, exclusive, memid, NULL, NULL, arena_id)) { - _mi_os_free_ex(subproc, start, size, commit, memid); - _mi_verbose_message("failed to reserve %zu KiB memory\n", _mi_divide_up(size, 1024)); - return ENOMEM; - } - _mi_verbose_message("reserved %zu KiB memory%s\n", _mi_divide_up(size, 1024), memid.is_pinned ? " (in large os pages)" : ""); - // mi_debug_show_arenas(true, true, false); - - return 0; -} - -// Reserve a range of regular OS memory -int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept { - return mi_reserve_os_memory_ex2(_mi_subproc(), size, commit, allow_large, exclusive, arena_id); -} - -// Manage a range of regular OS memory -bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept { - return mi_manage_os_memory_ex(start, size, is_committed, is_large, is_zero, numa_node, false /* exclusive? */, NULL); -} - -// Reserve a range of regular OS memory -int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept { - return mi_reserve_os_memory_ex(size, commit, allow_large, false, NULL); -} - - -/* ----------------------------------------------------------- - Debugging ------------------------------------------------------------ */ - -// Return idx of the slice past the last used slice -static size_t mi_arena_used_slices(mi_arena_t* arena) { - size_t idx; - if (mi_bbitmap_bsr_inv(arena->slices_free, &idx)) { - return (idx + 1); - } - else { - return mi_arena_info_slices(arena); - } -} - -static size_t mi_debug_show_bfield(mi_bfield_t field, char* buf, size_t* k) { - size_t bit_set_count = 0; - for (int bit = 0; bit < MI_BFIELD_BITS; bit++) { - bool is_set = ((((mi_bfield_t)1 << bit) & field) != 0); - if (is_set) bit_set_count++; - buf[*k] = (is_set ? 'x' : '.'); - *k = *k + 1; - } - return bit_set_count; -} - -typedef enum mi_ansi_color_e { - MI_BLACK = 30, - MI_MAROON, - MI_DARKGREEN, - MI_ORANGE, - MI_NAVY, - MI_PURPLE, - MI_TEAL, - MI_GRAY, - MI_DARKGRAY = 90, - MI_RED, - MI_GREEN, - MI_YELLOW, - MI_BLUE, - MI_MAGENTA, - MI_CYAN, - MI_WHITE -} mi_ansi_color_t; - -static void mi_debug_color(char* buf, size_t* k, mi_ansi_color_t color) { - *k += _mi_snprintf(buf + *k, 32, "\x1B[%dm", (int)color); -} - -static int mi_page_commit_usage(mi_page_t* page) { - const size_t committed_size = mi_page_committed(page); - const size_t used_size = mi_page_used(page) * mi_page_block_size(page); - return (int)(used_size * 100 / committed_size); -} - -static size_t mi_debug_show_page_bfield(char* buf, size_t* k, mi_arena_t* arena, size_t slice_index, long* pbit_of_page, mi_ansi_color_t* pcolor_of_page ) { - size_t bit_set_count = 0; - long bit_of_page = *pbit_of_page; - mi_ansi_color_t color = *pcolor_of_page; - mi_ansi_color_t prev_color = MI_GRAY; - for (int bit = 0; bit < MI_BFIELD_BITS; bit++, bit_of_page--) { - // bool is_set = ((((mi_bfield_t)1 << bit) & field) != 0); - void* start = mi_arena_slice_start(arena, slice_index + bit); - mi_page_t* page = _mi_safe_ptr_page(start); - char c = ' '; - if (page!=NULL && start==mi_page_slice_start(page)) { - mi_assert_internal(bit_of_page <= 0); - bit_set_count++; - c = 'p'; - color = MI_GRAY; - if (_mi_meta_is_meta_page(arena->subproc,page)) { c = 'm'; } - else if (mi_page_is_singleton(page)) { c = 's'; } - else if (mi_page_is_full(page)) { c = 'f'; } - if (!mi_page_is_abandoned(page)) { c = _mi_toupper(c); } - int commit_usage = mi_page_commit_usage(page); - if (commit_usage < 25) { color = MI_MAROON; } - else if (commit_usage < 50) { color = MI_ORANGE; } - else if (commit_usage < 75) { color = MI_TEAL; } - else color = MI_DARKGREEN; - bit_of_page = (long)page->memid.mem.arena.slice_count; - } - else { - c = '?'; - if (bit_of_page > 0) { c = '-'; } - // else if (_mi_meta_is_meta_page(arena->subproc,start)) { c = 'm'; color = MI_GRAY; } - else if (slice_index + bit < arena->info_slices) { c = 'i'; color = MI_GRAY; } - #if MI_PAGE_META_IS_ALIGNED - else if ((slice_index % MI_PAGE_META_ALIGNED_COUNT) == 0 && (size_t)bit <= mi_arena_page_meta_aligned_slice_count()) { - { c = 'i'; color = MI_GRAY; } - } - #endif - // else if (mi_bitmap_is_setN(arena->pages_purge, slice_index + bit, NULL)) { c = '*'; } - else if (mi_bbitmap_is_setN(arena->slices_free, slice_index+bit,1)) { - if (mi_bitmap_is_set(arena->slices_purge, slice_index + bit)) { c = '~'; color = MI_ORANGE; } - else if (mi_bitmap_is_set(arena->slices_committed, slice_index + bit)) { c = '_'; color = MI_GRAY; } - else { c = '.'; color = MI_GRAY; } - } - if (bit==MI_BFIELD_BITS-1 && bit_of_page > 1) { c = '>'; } - } - if (color != prev_color) { - mi_debug_color(buf, k, color); - prev_color = color; - } - buf[*k] = c; *k += 1; - } - mi_debug_color(buf, k, MI_GRAY); - *pbit_of_page = bit_of_page; - *pcolor_of_page = color; - return bit_set_count; -} - -static size_t mi_debug_show_chunks(const char* header1, const char* header2, const char* header3, - size_t slice_count, size_t chunk_count, - mi_bchunk_t* chunks, mi_bchunkmap_t* chunk_bins, bool invert, mi_arena_t* arena, bool narrow) -{ - _mi_raw_message("\x1B[37m%s%s%s (use/commit: \x1B[31m0 - 25%%\x1B[33m - 50%%\x1B[36m - 75%%\x1B[32m - 100%%\x1B[0m)\n", header1, header2, header3); - const size_t fields_per_line = (narrow ? 2 : 4); - const size_t used_slice_count = mi_arena_used_slices(arena); - size_t bit_count = 0; - size_t bit_set_count = 0; - long bit_of_page = 0; - mi_ansi_color_t color_of_page = MI_GRAY; - for (size_t i = 0; i < chunk_count && bit_count < slice_count; i++) { - char buf[5*MI_BCHUNK_BITS + 64]; _mi_memzero(buf, sizeof(buf)); - if (bit_count > used_slice_count && i+2 < chunk_count) { - const size_t diff = chunk_count - 1 - i; - bit_count += diff*MI_BCHUNK_BITS; - _mi_raw_message(" |\n"); - i = chunk_count-1; - } - - size_t k = 0; - - if (i<10) { buf[k++] = ('0' + (char)i); buf[k++] = ' '; buf[k++] = ' '; } - else if (i<100) { buf[k++] = ('0' + (char)(i/10)); buf[k++] = ('0' + (char)(i%10)); buf[k++] = ' '; } - else if (i<1000) { buf[k++] = ('0' + (char)(i/100)); buf[k++] = ('0' + (char)((i%100)/10)); buf[k++] = ('0' + (char)(i%10)); } - - char chunk_kind = ' '; - if (chunk_bins != NULL) { - switch (mi_bbitmap_debug_get_bin(chunk_bins,i)) { - case MI_CBIN_SMALL: chunk_kind = 'S'; break; - case MI_CBIN_MEDIUM: chunk_kind = 'M'; break; - case MI_CBIN_LARGE: chunk_kind = 'L'; break; - case MI_CBIN_HUGE: chunk_kind = 'H'; break; - case MI_CBIN_OTHER: chunk_kind = 'X'; break; - default: chunk_kind = ' '; break; // suppress warning - // case MI_CBIN_NONE: chunk_kind = 'N'; break; - } - } - buf[k++] = chunk_kind; - buf[k++] = ' '; - - for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) { - if (j > 0 && (j % fields_per_line) == 0) { - // buf[k++] = '\n'; _mi_memset(buf+k,' ',7); k += 7; - _mi_raw_message(" %s\n\x1B[37m", buf); - _mi_memzero(buf, sizeof(buf)); - _mi_memset(buf, ' ', 5); k = 5; - } - if (bit_count < slice_count) { - mi_bfield_t bfield = 0; - if (chunks!=NULL) { - bfield = chunks[i].bfields[j]; - } - if (invert) bfield = ~bfield; - size_t xcount = (chunks==NULL ? mi_debug_show_page_bfield(buf, &k, arena, bit_count, &bit_of_page, &color_of_page) - : mi_debug_show_bfield(bfield, buf, &k)); - if (invert) xcount = MI_BFIELD_BITS - xcount; - bit_set_count += xcount; - buf[k++] = ' '; - } - else { - _mi_memset(buf + k, 'o', MI_BFIELD_BITS); - k += MI_BFIELD_BITS; - } - bit_count += MI_BFIELD_BITS; - } - _mi_raw_message(" %s\n\x1B[37m", buf); - } - _mi_raw_message("\x1B[0m total pages: %zu\n", bit_set_count); - return bit_set_count; -} - -//static size_t mi_debug_show_bitmap_binned(const char* header1, const char* header2, const char* header3, size_t slice_count, -// mi_bitmap_t* bitmap, mi_bchunkmap_t* chunk_bins, bool invert, mi_arena_t* arena, bool narrow) { -// return mi_debug_show_chunks(header1, header2, header3, slice_count, mi_bitmap_chunk_count(bitmap), &bitmap->chunks[0], chunk_bins, invert, arena, narrow); -//} - -static void mi_debug_show_arenas_ex(mi_heap_t* heap, bool show_pages, bool narrow) mi_attr_noexcept { - mi_subproc_t* subproc = heap->subproc; - size_t max_arenas = mi_arenas_get_count(subproc); - //size_t free_total = 0; - //size_t slice_total = 0; - //size_t abandoned_total = 0; - size_t page_total = 0; - for (size_t i = 0; i < max_arenas; i++) { - mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]); - if (arena == NULL) continue; - mi_assert(arena->subproc == subproc); - // slice_total += arena->slice_count; - _mi_raw_message("%sarena %zu at %p: %zu slices (%zu MiB)%s%s, subproc: %zu, numa: %i\n", - (arena->parent==NULL ? "" : "(sub)"), i, arena, arena->slice_count, (size_t)(mi_size_of_slices(arena->slice_count)/MI_MiB), - (arena->memid.is_pinned ? ", pinned" : ""), (arena->is_exclusive ? ", exclusive" : ""), - arena->subproc->subproc_seq, arena->numa_node); - //if (show_inuse) { - // free_total += mi_debug_show_bbitmap("in-use slices", arena->slice_count, arena->slices_free, true, NULL); - //} - //if (show_committed) { - // mi_debug_show_bitmap("committed slices", arena->slice_count, arena->slices_committed, false, NULL); - //} - // todo: abandoned slices - //if (show_purge) { - // purge_total += mi_debug_show_bitmap("purgeable slices", arena->slice_count, arena->slices_purge, false, NULL); - //} - if (show_pages) { - // mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena); - // if (arena_pages != NULL) - { - const char* header1 = "chunks (p:page, f:full, s:single, m:meta-data, i:arena-info, P,F,S,M:not abandoned, ~:free-purgable, _:free-committed, .:free-reserved)"; - const char* header2 = (narrow ? "\n " : " "); - const char* header3 = "(chunk bin: S:small, M : medium, L : large, X : other)"; - page_total += mi_debug_show_chunks(header1, header2, header3, arena->slice_count, - mi_bbitmap_chunk_count(arena->slices_free), NULL, - arena->slices_free->chunkmap_bins, false, arena, narrow); - } - } - } - // if (show_inuse) _mi_raw_message("total inuse slices : %zu\n", slice_total - free_total); - // if (show_abandoned) _mi_raw_message("total abandoned slices: %zu\n", abandoned_total); - if (show_pages) _mi_raw_message("total pages in arenas: %zu\n", page_total); -} - -void mi_debug_show_arenas(void) mi_attr_noexcept { - mi_debug_show_arenas_ex(mi_heap_main(), true /* show pages */, true /* narrow? */); -} - -void mi_arenas_print(void) mi_attr_noexcept { - mi_debug_show_arenas(); -} - - -/* ----------------------------------------------------------- - Reserve a huge page arena. ------------------------------------------------------------ */ -// reserve at a specific numa node -int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept { - if (arena_id != NULL) *arena_id = NULL; - if (pages==0) return 0; - if (numa_node < -1) numa_node = -1; - if (numa_node >= 0) numa_node = numa_node % _mi_os_numa_node_count(); - mi_subproc_t* subproc = _mi_subproc(); - size_t hsize = 0; - size_t pages_reserved = 0; - mi_memid_t memid; - void* p = _mi_os_alloc_huge_os_pages(subproc, pages, numa_node, timeout_msecs, &pages_reserved, &hsize, &memid); - if (p==NULL || pages_reserved==0) { - _mi_warning_message("failed to reserve %zu GiB huge pages\n", pages); - return ENOMEM; - } - _mi_verbose_message("numa node %i: reserved %zu GiB huge pages (of the %zu GiB requested)\n", numa_node, pages_reserved, pages); - - if (!mi_manage_os_memory_ex2(subproc, p, hsize, numa_node, exclusive, memid, NULL, NULL, arena_id)) { - _mi_os_free(subproc, p, hsize, memid); - return ENOMEM; - } - return 0; -} - -int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept { - return mi_reserve_huge_os_pages_at_ex(pages, numa_node, timeout_msecs, false, NULL); -} - -// reserve huge pages evenly among the given number of numa nodes (or use the available ones as detected) -int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept { - if (pages == 0) return 0; - - // pages per numa node - int numa_count = (numa_nodes > 0 && numa_nodes <= INT_MAX ? (int)numa_nodes : _mi_os_numa_node_count()); - if (numa_count <= 0) { numa_count = 1; } - const size_t pages_per = pages / numa_count; - const size_t pages_mod = pages % numa_count; - const size_t timeout_per = (timeout_msecs==0 ? 0 : (timeout_msecs / numa_count) + 50); - - // reserve evenly among numa nodes - for (int numa_node = 0; numa_node < numa_count && pages > 0; numa_node++) { - size_t node_pages = pages_per; // can be 0 - if ((size_t)numa_node < pages_mod) { node_pages++; } - int err = mi_reserve_huge_os_pages_at(node_pages, numa_node, timeout_per); - if (err) return err; - if (pages < node_pages) { - pages = 0; - } - else { - pages -= node_pages; - } - } - - return 0; -} - -int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept { - MI_UNUSED(max_secs); - _mi_warning_message("mi_reserve_huge_os_pages is deprecated: use mi_reserve_huge_os_pages_interleave/at instead\n"); - if (pages_reserved != NULL) *pages_reserved = 0; - int err = mi_reserve_huge_os_pages_interleave(pages, 0, (size_t)(max_secs * 1000.0)); - if (err==0 && pages_reserved!=NULL) *pages_reserved = pages; - return err; -} - - - - - -/* ----------------------------------------------------------- - Arena purge ------------------------------------------------------------ */ - -static long mi_arena_purge_delay(void) { - // <0 = no purging allowed, 0=immediate purging, >0=milli-second delay - const long delay = mi_option_get(mi_option_purge_delay); - const long mult = mi_option_get(mi_option_arena_purge_mult); - if (delay<0 || mult<0) { return -1; } - if (delay==0 || mult==0) { return 0; } - size_t total; - if (mi_mul_overflow((size_t)delay, (size_t)mult, &total)) { return delay; } - if (total > LONG_MAX) { return delay; } - return (long)total; -} - -// reset or decommit in an arena and update the commit bitmap -// assumes we own the area (i.e. slices_free is claimed by us) -// returns if the memory is no longer committed (versus reset which keeps the commit) -static bool mi_arena_purge(mi_arena_t* arena, size_t slice_index, size_t slice_count) { - mi_assert_internal(!arena->memid.is_pinned); - mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); - - const size_t size = mi_size_of_slices(slice_count); - void* const p = mi_arena_slice_start(arena, slice_index); - //const bool all_committed = mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count); - size_t already_committed; - mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed); // pretend all committed.. (as we lack a clearN call that counts the already set bits..) - const bool all_committed = (already_committed == slice_count); - const bool needs_recommit = _mi_os_purge_ex(arena->subproc, p, size, all_committed /* allow reset? */, mi_size_of_slices(already_committed), arena->commit_fun, arena->commit_fun_arg); - - if (needs_recommit) { - // no longer committed - mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count); - // we just counted in the purge to decommit all, but the some part was not committed so adjust that here - // mi_subproc_stat_decrease(arena->subproc, committed, mi_size_of_slices(slice_count - already_committed)); - } - else if (!all_committed) { - // we cannot assume any of these are committed any longer (even with reset since we did setN and may have marked uncommitted slices as committed) - mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count); - // we adjust the commit count as parts will be re-committed - // mi_subproc_stat_decrease(arena->subproc, committed, mi_size_of_slices(already_committed)); - } - - return needs_recommit; -} - - -// Schedule a purge. This is usually delayed to avoid repeated decommit/commit calls. -// Note: assumes we (still) own the area as we may purge immediately -static void mi_arena_schedule_purge(mi_arena_t* arena, size_t slice_index, size_t slice_count) { - const long delay = mi_arena_purge_delay(); - if (arena->memid.is_pinned || delay < 0 || _mi_preloading()) return; // is purging allowed at all? - - mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); - if (delay == 0) { - // purge directly - mi_arena_purge(arena, slice_index, slice_count); - } - else { - // schedule purge - const mi_msecs_t expire = _mi_clock_now() + delay; - mi_msecs_t expire0 = 0; - if (mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &expire0, expire)) { - // expiration was not yet set - // maybe set the global arenas expire as well (if it wasn't set already) - mi_assert_internal(expire0==0); - mi_atomic_casi64_strong_acq_rel(&arena->subproc->purge_expire, &expire0, expire); - } - else { - // already an expiration was set - } - mi_bitmap_setN(arena->slices_purge, slice_index, slice_count, NULL); - } -} - -typedef struct mi_purge_visit_info_s { - mi_msecs_t now; - mi_msecs_t delay; - bool all_purged; - bool any_purged; -} mi_purge_visit_info_t; - -static bool mi_arena_try_purge_range(mi_arena_t* arena, size_t slice_index, size_t slice_count) { - mi_assert(slice_count < MI_BCHUNK_BITS); - if (mi_bbitmap_try_clearNC(arena->slices_free, slice_index, slice_count)) { - // purge - bool decommitted = mi_arena_purge(arena, slice_index, slice_count); MI_UNUSED(decommitted); - mi_assert_internal(!decommitted || mi_bitmap_is_clearN(arena->slices_committed, slice_index, slice_count)); - // and reset the free range - mi_bbitmap_setN(arena->slices_free, slice_index, slice_count); - return true; - } - else { - // was allocated again already - return false; - } -} - -static bool mi_arena_try_purge_visitor(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) { - mi_purge_visit_info_t* vinfo = (mi_purge_visit_info_t*)arg; - // try to purge: first claim the free blocks - if (mi_arena_try_purge_range(arena, slice_index, slice_count)) { - vinfo->any_purged = true; - vinfo->all_purged = true; - } - else if (slice_count > 1) - { - // failed to claim the full range, try per slice instead - for (size_t i = 0; i < slice_count; i++) { - const bool purged = mi_arena_try_purge_range(arena, slice_index + i, 1); - vinfo->any_purged = vinfo->any_purged || purged; - vinfo->all_purged = vinfo->all_purged && purged; - } - } - // don't clear the purge bits as that is done atomically be the _bitmap_forall_set_ranges - // mi_bitmap_clearN(arena->slices_purge, slice_index, slice_count); - return true; // continue -} - -// returns -// -1 = nothing was purged -// 0 = nothing was purged yet because have not yet reached the expire time -// 1 = some pages in the arena were purged -static int mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force) -{ - // check pre-conditions - if (arena->memid.is_pinned) return -1; - - // expired yet? - mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire); - if (expire==0) return -1; - if (!force && expire > now) return 0; - - // reset expire - mi_atomic_storei64_release(&arena->purge_expire, (mi_msecs_t)0); - mi_subproc_stat_counter_increase(arena->subproc, arena_purges, 1); - - // go through all purge info's (with max MI_BFIELD_BITS ranges at a time) - // this also clears those ranges atomically (so any newly freed blocks will get purged next - // time around) - mi_purge_visit_info_t vinfo = { now, mi_arena_purge_delay(), true /*all?*/, false /*any?*/}; - - // we purge by at least `minslices` to not fragment transparent huge pages for example - const size_t minslices = mi_slice_count_of_size(_mi_os_minimal_purge_size()); - _mi_bitmap_forall_setc_rangesn(arena->slices_purge, minslices, &mi_arena_try_purge_visitor, arena, &vinfo); - - return (vinfo.any_purged ? 1 : -1); -} - - -static void mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq) -{ - // try purge can be called often so try to only run when needed - const long delay = mi_arena_purge_delay(); - if (_mi_preloading() || delay <= 0) return; // nothing will be scheduled - - // check if any arena needs purging? - const mi_msecs_t now = _mi_clock_now(); - const mi_msecs_t arenas_expire = mi_atomic_loadi64_acquire(&subproc->purge_expire); - if (!visit_all && !force && (arenas_expire == 0 || arenas_expire > now)) return; - - const size_t max_arena = mi_arenas_get_count(subproc); - if (max_arena == 0) return; - - // allow only one thread to purge at a time (todo: allow concurrent purging?) - static mi_atomic_guard_t purge_guard; - mi_atomic_guard(&purge_guard) - { - // increase global expire: at most one purge per delay cycle - if (arenas_expire > now) { mi_atomic_storei64_release(&subproc->purge_expire, now + (delay/10)); } - const size_t arena_start = tseq % max_arena; - size_t max_purge_count = (visit_all ? max_arena : (max_arena/4)+1); - bool all_visited = true; - bool any_purged = false; - for (size_t _i = 0; _i < max_arena; _i++) { - size_t i = _i + arena_start; - if (i >= max_arena) { i -= max_arena; } - mi_arena_t* arena = mi_arena_from_index(subproc,i); - if (arena != NULL) { - const int purged = mi_arena_try_purge(arena, now, force); - if (purged >= 0) { // purged, or arena expire is not yet reached - any_purged = true; - if (purged >= 1) { // purged - if (max_purge_count <= 1) { - all_visited = false; - break; - } - max_purge_count--; - } - } - } - } - if (all_visited && !any_purged) { - mi_atomic_storei64_release(&subproc->purge_expire, (mi_msecs_t)0); - } - } -} - - -/* ----------------------------------------------------------- - Visit all pages and blocks in a heap ------------------------------------------------------------ */ - -typedef struct mi_heap_visit_info_s { - mi_heap_t* heap; - mi_block_visit_fun* visitor; - void* arg; - bool visit_blocks; -} mi_heap_visit_info_t; - -static bool mi_heap_visit_page(mi_page_t* page, mi_heap_visit_info_t* vinfo) { - mi_heap_area_t area; - _mi_heap_area_init(&area, page); - mi_assert_internal(vinfo->heap == mi_page_heap(page)); - if (!vinfo->visitor(vinfo->heap, &area, NULL, area.block_size, vinfo->arg)) { - return false; - } - if (vinfo->visit_blocks) { - return _mi_theap_area_visit_blocks(&area, page, vinfo->visitor, vinfo->arg); - } - else { - return true; - } -} - -static bool mi_heap_visit_page_at(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) { - MI_UNUSED(slice_count); - mi_heap_visit_info_t* vinfo = (mi_heap_visit_info_t*)arg; - mi_page_t* page = mi_arena_page_at_slice(arena, slice_index); - return mi_heap_visit_page(page, vinfo); -} - -bool _mi_heap_visit_blocks(mi_heap_t* heap, bool abandoned_only, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { - mi_assert(visitor!=NULL); - if (visitor==NULL) return false; - if (heap==NULL) { heap = mi_heap_main(); } - // visit all pages in a heap - // we don't have to claim because we assume we are the only thread running (with this heap). - // (but we could atomically claim as well by first doing abandoned_reclaim and afterwards reabandoning). - mi_heap_visit_info_t visit_info = { heap, visitor, arg, visit_blocks }; - bool ok = true; - mi_forall_arenas(heap, NULL, 0, arena) { - mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena); - if (ok && arena_pages != NULL) { - if (abandoned_only) { - for (size_t bin = 0; ok && bin < MI_ARENA_BIN_COUNT; bin++) { - // todo: if we had a single abandoned page map as well, this can be faster. - if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) > 0) { - ok = _mi_bitmap_forall_set(arena_pages->pages_abandoned[bin], &mi_heap_visit_page_at, arena, &visit_info); - } - } - } - else { - ok = _mi_bitmap_forall_set(arena_pages->pages, &mi_heap_visit_page_at, arena, &visit_info); - } - } - } - mi_forall_arenas_end(); - if (!ok) return false; - - // visit abandoned pages in OS allocated memory - // (technically we don't need the initial lock as we assume we are the only thread running in this subproc) - mi_page_t* page = NULL; - mi_lock(&heap->os_abandoned_pages_lock) { - page = heap->os_abandoned_pages; - } - while (ok && page != NULL) { - mi_page_t* next = page->next; // read upfront in case the visitor frees the page - ok = mi_heap_visit_page(page, &visit_info); - page = next; - } - - return ok; -} - -bool mi_heap_visit_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { - return _mi_heap_visit_blocks(heap, false, visit_blocks, visitor, arg); -} - -bool mi_heap_visit_abandoned_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { - return _mi_heap_visit_blocks(heap, true, visit_blocks, visitor, arg); -} - - -typedef struct mi_heap_delete_visit_info_s { - mi_heap_t* heap_target; - mi_theap_t* theap_target; - mi_theap_t* theap; -} mi_heap_delete_visit_info_t; - -static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg) { - MI_UNUSED(block); MI_UNUSED(block_size); MI_UNUSED(heap); - mi_heap_delete_visit_info_t* info = (mi_heap_delete_visit_info_t*)arg; - mi_heap_t* heap_target = info->heap_target; - mi_theap_t* const theap = NULL; // info->theap; mi_assert_internal(_mi_theap_heap(theap) == heap); - mi_page_t* const page = (mi_page_t*)area->reserved1; - - mi_page_claim_ownership(page); // claim ownership - if (mi_page_is_abandoned(page)) { - _mi_arenas_page_unabandon(page,theap); - } - else { - page->next = page->prev = NULL; // yikes.. better not to try to access this from a thread later on.. - mi_page_set_theap(page,NULL); // set threadid to abandoned - } - mi_assert_internal(mi_page_is_abandoned(page)); - mi_assert_internal(mi_page_is_owned(page)); - - if (mi_page_used(page)==0) { - // free the page - _mi_arenas_page_free(page, theap); - } - else if (heap_target==NULL) { - #if MI_GUARDED - _mi_page_unguard_all(page); // remove potential interior guard pages - #endif - // destroy the page - mi_page_used_reset(page); // note: invariant `|local_free| + |free| == reserved - used` does not hold in this case - _mi_arenas_page_free(page, theap); - } - else { - // move the page to `heap_target` as an abandoned page - // first remove it from the current heap - const size_t sbin = _mi_page_stats_bin(page); - mi_arena_t* arena = NULL; - size_t slice_index = 0; - if (page->memid.memkind == MI_MEM_ARENA) { - size_t slice_count; - mi_arena_pages_t* arena_pages = NULL; - arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); - mi_assert_internal(mi_bitmap_is_set(arena_pages->pages, slice_index)); - mi_bitmap_clear(arena_pages->pages, slice_index); - } - else { - // os allocated - mi_assert_internal(mi_memid_is_os(page->memid) && page->next == NULL); - } - if (theap != NULL) { - mi_theap_stat_decrease(theap, page_bins[sbin], 1); - mi_theap_stat_decrease(theap, pages, 1); - } - else { - mi_heap_stat_decrease((mi_heap_t*)heap, page_bins[_mi_page_stats_bin(page)], 1); - mi_heap_stat_decrease((mi_heap_t*)heap, pages, 1); - } - mi_theap_t* theap_target = info->theap_target; - - // and then add it to the new target heap - if (arena != NULL) { - mi_arena_pages_t* arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena); - if mi_unlikely(arena_pages_target==NULL) { - // if we cannot allocate this, we move it to the main heap instead (which does not require allocation) - heap_target = mi_arena_heap_main(arena); - theap_target = mi_heap_theap(heap_target); // todo: find through theap_target tld? - arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena); - mi_assert_internal(arena_pages_target!=NULL); - } - mi_assert_internal(mi_bitmap_is_clear(arena_pages_target->pages, slice_index)); - mi_bitmap_set(arena_pages_target->pages, slice_index); - } - page->heap = heap_target; - mi_theap_stat_increase(theap_target, page_bins[sbin], 1); - mi_theap_stat_increase(theap_target, pages, 1); - - // and abandon in the new heap - _mi_arenas_page_abandon(page,theap_target); - } - return true; -} - -static void mi_heap_delete_pages(mi_heap_t* heap, mi_heap_t* heap_target) { - mi_theap_t* const theap_target = (heap_target != NULL ? _mi_heap_theap(heap_target) : NULL); - // mi_theap_t* const theap = _mi_heap_theap(heap); - mi_heap_delete_visit_info_t info = { heap_target, theap_target, NULL }; - _mi_heap_visit_blocks(heap, false, false, &mi_heap_delete_page, &info); - #if MI_DEBUG>1 - // no more arena pages? - for (size_t i = 0; i < MI_MAX_ARENAS; i++) { - mi_arena_pages_t* const arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]); - if (arena_pages!=NULL) { - mi_assert_internal(mi_bitmap_is_all_clear(arena_pages->pages)); - } - } - // nor os abandoned pages? - mi_lock(&heap->os_abandoned_pages_lock) { - - mi_assert_internal(heap->os_abandoned_pages == NULL); - } - // nor arena abandoned pages? - for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) { - mi_assert_internal(mi_atomic_load_relaxed(&heap->abandoned_count[i])==0); - } - #endif -} - -void _mi_heap_move_pages(mi_heap_t* heap_from, mi_heap_t* heap_to) { - if (_mi_is_heap_main(heap_from)) return; - if (heap_to==NULL) { heap_to = heap_from->subproc->heap_main; } - mi_heap_delete_pages(heap_from, heap_to); -} - -void _mi_heap_destroy_pages(mi_heap_t* heap_from) { - if (_mi_is_heap_main(heap_from)) return; - mi_heap_delete_pages(heap_from, NULL); -} - -/* ----------------------------------------------------------- - Unloading and reloading an arena. ------------------------------------------------------------ */ -/* -static bool mi_arena_page_register(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) { - MI_UNUSED(arg); MI_UNUSED(slice_count); - mi_assert_internal(slice_count == 1); - mi_page_t* page = mi_arena_page_at_slice(arena, slice_index); - mi_assert_internal(mi_bitmap_is_setN(page->memid.mem.arena.arena->pages, page->memid.mem.arena.slice_index, 1)); - if (!_mi_page_map_register(page)) return false; // break - mi_assert_internal(_mi_ptr_page(page)==page); - return true; -} - -mi_decl_nodiscard static bool mi_arena_pages_reregister(mi_arena_t* arena) { - return _mi_bitmap_forall_set(arena->pages, &mi_arena_page_register, arena, NULL); -} - -mi_decl_export bool mi_arena_unload(mi_arena_id_t arena_id, void** base, size_t* accessed_size, size_t* full_size) { - mi_arena_t* arena = _mi_arena_from_id(arena_id); - if (arena==NULL) { - return false; - } - else if (!arena->is_exclusive) { - _mi_warning_message("cannot unload a non-exclusive arena (id %zu at %p)\n", arena_id, arena); - return false; - } - else if (arena->memid.memkind != MI_MEM_EXTERNAL) { - _mi_warning_message("can only unload managed arena's for external memory (id %zu at %p)\n", arena_id, arena); - return false; - } - - // find accessed size - const size_t asize = mi_size_of_slices(mi_arena_used_slices(arena)); - if (base != NULL) { *base = (void*)arena; } - if (full_size != NULL) { *full_size = arena->memid.mem.os.size; } - if (accessed_size != NULL) { *accessed_size = asize; } - - // adjust abandoned page count - mi_subproc_t* const subproc = arena->subproc; - for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) { - const size_t count = mi_bitmap_popcount(arena->pages_abandoned[bin]); - if (count > 0) { mi_atomic_decrement_acq_rel(&subproc->abandoned_count[bin]); } - } - - // unregister the pages - _mi_page_map_unregister_range(arena, asize); - - // set arena entry to NULL - const size_t count = mi_arenas_get_count(subproc); - for(size_t i = 0; i < count; i++) { - if (mi_arena_from_index(subproc, i) == arena) { - mi_atomic_store_ptr_release(mi_arena_t, &subproc->arenas[i], NULL); - if (i + 1 == count) { // try adjust the count? - size_t expected = count; - mi_atomic_cas_strong_acq_rel(&subproc->arena_count, &expected, count-1); - } - break; - } - } - return true; -} - -mi_decl_export bool mi_arena_reload(void* start, size_t size, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) { - // assume the memory area is already containing the arena - if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); } - if (start == NULL || size == 0) return false; - mi_arena_t* arena = (mi_arena_t*)start; - mi_memid_t memid = arena->memid; - if (memid.memkind != MI_MEM_EXTERNAL) { - _mi_warning_message("can only reload arena's from external memory (%p)\n", arena); - return false; - } - if (memid.mem.os.base != start) { - _mi_warning_message("the reloaded arena base address differs from the external memory (arena: %p, external: %p)\n", arena, start); - return false; - } - if (memid.mem.os.size != size) { - _mi_warning_message("the reloaded arena size differs from the external memory (arena size: %zu, external size: %zu)\n", arena->memid.mem.os.size, size); - return false; - } - if (!arena->is_exclusive) { - _mi_warning_message("the reloaded arena is not exclusive\n"); - return false; - } - - // re-initialize - arena->is_exclusive = true; - arena->commit_fun = commit_fun; - arena->commit_fun_arg = commit_fun_arg; - arena->subproc = _mi_subproc(); - if (!mi_arenas_add(arena->subproc, arena, arena_id)) { - return false; - } - if (!mi_arena_pages_reregister(arena)) { - // todo: clear arena entry in the subproc? - return false; - } - - // adjust abandoned page count - for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) { - const size_t count = mi_bitmap_popcount(arena->pages_abandoned[bin]); - if (count > 0) { mi_atomic_decrement_acq_rel(&arena->subproc->abandoned_count[bin]); } - } - - return true; -} - -*/ diff --git a/system/lib/mimalloc/src/arena/arena-alloc.c b/system/lib/mimalloc/src/arena/arena-alloc.c new file mode 100644 index 0000000000000..6b652c4cd4b4a --- /dev/null +++ b/system/lib/mimalloc/src/arena/arena-alloc.c @@ -0,0 +1,554 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#include "mimalloc.h" +#include "mimalloc/util/stats.h" +#include "mimalloc/alloc/free.h" // mi_free_subproc_safe +#include "mimalloc/alloc/page.h" +#include "mimalloc/heap/subproc.h" +#include "mimalloc/arena/arena-alloc.h" +#include "arena.h" + + +mi_decl_nodiscard bool _mi_arena_commit(mi_subproc_t* subproc, mi_arena_t* arena, void* start, size_t size, bool* is_zero, size_t stat_already_committed) { + mi_assert_internal(subproc!=NULL); + if (arena != NULL && arena->commit_fun != NULL) { + return (*arena->commit_fun)(true, start, size, is_zero, arena->commit_fun_arg); + } + else if (stat_already_committed > 0) { + return _mi_os_commit_ex(subproc, start, size, is_zero, stat_already_committed); + } + else { + return _mi_os_commit(subproc, start, size, is_zero); + } +} + +// Create an arena memid +static mi_memid_t mi_memid_create_arena(mi_arena_t* arena, size_t slice_index, size_t slice_count) { + mi_assert_internal(slice_index < UINT32_MAX); + mi_assert_internal(slice_count < UINT32_MAX); + mi_assert_internal(slice_count > 0); + mi_assert_internal(slice_index < arena->slice_count); + mi_memid_t memid = _mi_memid_create(MI_MEM_ARENA); + memid.mem.arena.arena = arena; + memid.mem.arena.slice_index = (uint32_t)slice_index; + memid.mem.arena.slice_count = (uint32_t)slice_count; + return memid; +} + +/* ----------------------------------------------------------- + Arena Allocation +----------------------------------------------------------- */ + +static mi_decl_noinline void* mi_arena_try_alloc_at( + mi_arena_t* arena, size_t slice_count, bool commit, size_t tseq, mi_memid_t* memid) +{ + mi_assert_internal(arena!=NULL); + mi_assert_internal(slice_count>0); + size_t slice_index; + if (!mi_bbitmap_try_find_and_clearN(arena->slices_free, tseq, slice_count, &slice_index)) return NULL; + + // claimed it! + void* p = mi_arena_slice_start(arena, slice_index); + *memid = mi_memid_create_arena(arena, slice_index, slice_count); + memid->is_pinned = arena->memid.is_pinned; + + // set the dirty bits and track which slices become accessible + size_t touched_slices = slice_count; + if (arena->memid.initially_zero) { + size_t already_dirty = 0; + memid->initially_zero = mi_bitmap_setN(arena->slices_dirty, slice_index, slice_count, &already_dirty); + mi_assert_internal(already_dirty <= touched_slices); + touched_slices -= already_dirty; + } + else { + // todo: properly count touched pages with a separate bitmap? + touched_slices = 0; + } + + // set commit state + if (commit) { + // commit requested, but the range may not be committed as a whole: ensure it is committed now + const size_t already_committed = mi_bitmap_popcountN(arena->slices_committed, slice_index, slice_count); + if (already_committed < slice_count) { + // not all committed, try to commit now + bool commit_zero = false; + if (!_mi_arena_commit(arena->subproc, arena, p, _mi_size_of_slices(slice_count), &commit_zero, _mi_size_of_slices(already_committed))) { + // if the commit fails, release ownership, and return NULL; + // note: this does not roll back dirty bits but that is ok. + mi_bbitmap_setN(arena->slices_free, slice_index, slice_count); + return NULL; + } + if (commit_zero) { + memid->initially_zero = true; + } + + // set the commit bits + mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, NULL); + + // committed + #if MI_DEBUG > 1 + if (memid->initially_zero) { + if (!mi_mem_is_zero(p, _mi_size_of_slices(slice_count))) { + _mi_error_message(EFAULT, "internal error: arena allocation was not zero-initialized!\n"); + memid->initially_zero = false; + } + } + #endif + } + else { + // already fully committed. + _mi_os_reuse(arena->subproc, p, _mi_size_of_slices(slice_count)); + // if the OS has overcommit, and this is the first time we access these pages, then + // count the commit now (as at arena reserve we didn't count those commits as these are on-demand) + if (_mi_os_has_overcommit() && touched_slices > 0 && !arena->memid.is_pinned /* huge pages, issue #1236 */) { + mi_subproc_stat_increase( arena->subproc, committed, _mi_size_of_slices(touched_slices)); + } + } + + mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + memid->initially_committed = true; + + // tool support + if (memid->initially_zero) { + mi_track_mem_defined(p, slice_count * MI_ARENA_SLICE_SIZE); + } + else { + mi_track_mem_undefined(p, slice_count * MI_ARENA_SLICE_SIZE); + } + } + else { + // no need to commit, but check if it is already fully committed + memid->initially_committed = mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count); + if (!memid->initially_committed) { + // partly committed.. adjust stats + size_t already_committed_count = 0; + mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed_count); + mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count); + mi_subproc_stat_decrease(arena->subproc, committed, _mi_size_of_slices(already_committed_count)); + } + } + + mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); + if (commit) { mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); } + if (commit) { mi_assert_internal(memid->initially_committed); } + mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count)); + + return p; +} + +/* ----------------------------------------------------------- + Arena allocation +----------------------------------------------------------- */ + +typedef struct mi_arena_alloc_visit_info_s { + size_t slice_count; + bool commit; + size_t tseq; + mi_memid_t* memid; +} mi_arena_alloc_visit_info_t; + +static bool mi_arena_try_alloc_visit(mi_arena_t* arena, const void* arg, void** result) { + const mi_arena_alloc_visit_info_t* info = (const mi_arena_alloc_visit_info_t*)arg; + mi_assert_internal(result != NULL); + *result = mi_arena_try_alloc_at(arena, info->slice_count, info->commit, info->tseq, info->memid); + if (*result != NULL) return false; // break + return true; // continue +} + +// allocate slices from the arenas +static mi_decl_noinline void* mi_arenas_try_find_free( + mi_heap_t* heap, size_t slice_count, size_t alignment, + bool commit, bool allow_large, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid) +{ + // mi_assert_internal(slice_count <= _mi_slice_count_of_size(MI_ARENA_MAX_CHUNK_OBJ_SIZE)); + mi_assert(alignment <= MI_ARENA_SLICE_ALIGN); + if (alignment > MI_ARENA_SLICE_ALIGN) return NULL; + + // search arena's + const mi_arena_alloc_visit_info_t info = { slice_count, commit, tseq, memid }; + void* p = NULL; + if (!mi_forall_suitable_arenas(heap, req_arena, tseq, true /* only numa matching */, numa_node, allow_large, &mi_arena_try_alloc_visit, &info, &p)) { + return p; + } + if (numa_node < 0) return NULL; + + // search again but now regardless of preferred numa affinity + mi_forall_suitable_arenas(heap, req_arena, tseq, false /* numa non-matching now */, numa_node, allow_large, &mi_arena_try_alloc_visit, &info, &p); + return p; +} + +// Allocate slices from the arena's -- potentially allocating a fresh arena +mi_decl_noinline void* _mi_arenas_try_alloc( + mi_heap_t* heap, + size_t slice_count, size_t alignment, + bool commit, bool allow_large, + mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid) +{ + // mi_assert(slice_count <= MI_ARENA_MAX_CHUNK_OBJ_SLICES); + mi_assert(alignment <= MI_ARENA_SLICE_ALIGN); + void* p; + + // not too large? + if (slice_count * MI_ARENA_SLICE_SIZE > MI_ARENA_MAX_SIZE) return NULL; + + // try to find free slices in the arena's + p = mi_arenas_try_find_free(heap, slice_count, alignment, commit, allow_large, req_arena, tseq, numa_node, memid); + if (p != NULL) return p; + + // did we need a specific arena? + if (req_arena != NULL) return NULL; + + // don't create arena's if OS allocation is disallowed + if (mi_option_is_enabled(mi_option_disallow_os_alloc)) return NULL; + + // otherwise, try to reserve a new arena -- but one thread at a time.. (todo: allow 2 or 4 to reduce contention?) + mi_subproc_t* const subproc = heap->subproc; + const size_t arena_count = mi_arenas_get_count(subproc); + mi_lock(&subproc->arena_reserve_lock) { + if (arena_count == mi_arenas_get_count(subproc)) { + // we are the first to enter the lock, reserve a fresh arena + mi_arena_id_t arena_id = _mi_arena_id_none(); + _mi_arena_reserve(subproc, _mi_size_of_slices(slice_count), allow_large, &arena_id); + } + else { + // another thread already reserved a new arena + } + } + // try once more to allocate in the new arena + mi_assert_internal(req_arena == NULL); + p = mi_arenas_try_find_free(heap, slice_count, alignment, commit, allow_large, req_arena, tseq, numa_node, memid); + if (p != NULL) return p; + + return NULL; +} + +// Allocate from the OS (if allowed) +void* _mi_arena_os_alloc_aligned( + mi_subproc_t* subproc, + size_t size, size_t alignment, size_t align_offset, + bool commit, bool allow_large, + mi_arena_id_t req_arena_id, mi_memid_t* memid) +{ + // if we cannot use OS allocation, return NULL + if (mi_option_is_enabled(mi_option_disallow_os_alloc) || req_arena_id != _mi_arena_id_none()) { + errno = ENOMEM; + return NULL; + } + + if (align_offset > 0) { + return _mi_os_alloc_aligned_at_offset(subproc, size, alignment, align_offset, commit, allow_large, memid); + } + else { + return _mi_os_alloc_aligned(subproc, size, alignment, commit, allow_large, memid); + } +} + + +// Allocate large sized memory +void* _mi_arenas_alloc_aligned( mi_heap_t* heap, + size_t size, size_t alignment, size_t align_offset, + bool commit, bool allow_large, + mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid) +{ + mi_assert_internal(memid != NULL); + mi_assert_internal(size > 0); + + // try to allocate in an arena if the alignment is small enough and the object is not too small (as for theap meta data) + if (!mi_option_is_enabled(mi_option_disallow_arena_alloc) && // is arena allocation allowed? + size >= MI_ARENA_MIN_OBJ_SIZE && size <= mi_arena_max_object_size() && // and not too small or too large + alignment <= MI_ARENA_SLICE_ALIGN && align_offset == 0) // and good alignment + { + const size_t slice_count = _mi_slice_count_of_size(size); + void* p = _mi_arenas_try_alloc(heap, slice_count, alignment, commit, allow_large, req_arena, tseq, numa_node, memid); + if (p != NULL) return p; + } + + // fall back to the OS + void* p = _mi_arena_os_alloc_aligned(heap->subproc, size, alignment, align_offset, commit, allow_large, req_arena, memid); + return p; +} + +void* _mi_arenas_alloc(mi_heap_t* heap, size_t size, bool commit, bool allow_large, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid) +{ + return _mi_arenas_alloc_aligned(heap, size, MI_ARENA_SLICE_SIZE, 0, commit, allow_large, req_arena, tseq, numa_node, memid); +} + +/* ----------------------------------------------------------- + Arena free +----------------------------------------------------------- */ +static void mi_arena_schedule_purge(mi_arena_t* arena, size_t slice_index, size_t slices); +static void mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq); + +void _mi_arenas_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid) { + if (p==NULL) return; + if (size==0) return; + + // need to set all memory to undefined as some parts may still be marked as no_access (like padding etc.) + mi_track_mem_undefined(p, size); + + if (mi_memkind_is_os(memid.memkind)) { + // was a direct OS allocation, pass through + _mi_os_free(subproc, p, size, memid); + } + else if (memid.memkind == MI_MEM_ARENA) { + // allocated in an arena + size_t slice_count; + size_t slice_index; + mi_arena_t* arena = mi_arena_from_memid(memid, &slice_index, &slice_count); + mi_assert_internal(arena!=NULL); + mi_assert_internal(arena->subproc == subproc); + mi_assert_internal((size%MI_ARENA_SLICE_SIZE)==0); + mi_assert_internal((slice_count*MI_ARENA_SLICE_SIZE)==size); + mi_assert_internal(mi_arena_slice_start(arena,slice_index) <= (uint8_t*)p); + mi_assert_internal(mi_arena_slice_start(arena,slice_index) + _mi_size_of_slices(slice_count) > (uint8_t*)p); + // checks + if (arena == NULL) { + _mi_error_message(EINVAL, "trying to free from an invalid arena: %p, size %zu, memkind: 0x%x\n", p, size, memid.memkind); + return; + } + mi_assert_internal(slice_index < arena->slice_count); + mi_assert_internal(slice_index >= mi_arena_info_slices(arena)); + if (slice_index < mi_arena_info_slices(arena) || slice_index >= arena->slice_count) { + _mi_error_message(EINVAL, "trying to free from an invalid arena block: %p, size %zu, memkind: 0x%x\n", p, size, memid.memkind); + return; + } + + // potentially decommit + if (!arena->memid.is_pinned /* && !arena->memid.initially_committed */) { // todo: allow decommit even if initially committed? + // (delay) purge the page + mi_arena_schedule_purge(arena, slice_index, slice_count); + } + + // and make it available to others again + bool all_inuse = mi_bbitmap_setN(arena->slices_free, slice_index, slice_count); + if (!all_inuse) { + _mi_error_message(EAGAIN, "trying to free an already freed arena block: %p, size %zu\n", mi_arena_slice_start(arena,slice_index), _mi_size_of_slices(slice_count)); + return; + }; + } + else if (memid.memkind == MI_MEM_MALLOC) { + _mi_free_subproc_safe(p); + } + else { + // arena was none, external, or static; nothing to do + mi_assert_internal(mi_memid_needs_no_free(memid)); + } + + // try to purge expired decommits + #if MI_ARENA_PURGE_ON_FREE + mi_arenas_try_purge(false, false, subproc, 0); + #endif +} + +// Purge the arenas; if `force_purge` is true, amenable parts are purged even if not yet expired +void _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld) { + mi_arenas_try_purge(force_purge, visit_all, tld->subproc, tld->thread_seq); +} + +/* ----------------------------------------------------------- + Arena purge +----------------------------------------------------------- */ + +static long mi_arena_purge_delay(void) { + // <0 = no purging allowed, 0=immediate purging, >0=milli-second delay + const long delay = mi_option_get(mi_option_purge_delay); + const long mult = mi_option_get(mi_option_arena_purge_mult); + if (delay<0 || mult<0) { return -1; } + if (delay==0 || mult==0) { return 0; } + size_t total; + if (mi_mul_overflow((size_t)delay, (size_t)mult, &total)) { return delay; } + if (total > LONG_MAX) { return delay; } + return (long)total; +} + +// reset or decommit in an arena and update the commit bitmap +// assumes we own the area (i.e. slices_free is claimed by us) +// returns if the memory is no longer committed (versus reset which keeps the commit) +static bool mi_arena_purge(mi_arena_t* arena, size_t slice_index, size_t slice_count) { + mi_assert_internal(!arena->memid.is_pinned); + mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); + + const size_t size = _mi_size_of_slices(slice_count); + void* const p = mi_arena_slice_start(arena, slice_index); + //const bool all_committed = mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count); + size_t already_committed; + mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed); // pretend all committed.. (as we lack a clearN call that counts the already set bits..) + const bool all_committed = (already_committed == slice_count); + const bool needs_recommit = _mi_os_purge_ex(arena->subproc, p, size, all_committed /* allow reset? */, _mi_size_of_slices(already_committed), arena->commit_fun, arena->commit_fun_arg); + + if (needs_recommit) { + // no longer committed + mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count); + // we just counted in the purge to decommit all, but the some part was not committed so adjust that here + // mi_subproc_stat_decrease(arena->subproc, committed, _mi_size_of_slices(slice_count - already_committed)); + } + else if (!all_committed) { + // we cannot assume any of these are committed any longer (even with reset since we did setN and may have marked uncommitted slices as committed) + mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count); + // we adjust the commit count as parts will be re-committed + // mi_subproc_stat_decrease(arena->subproc, committed, _mi_size_of_slices(already_committed)); + } + + return needs_recommit; +} + + +// Schedule a purge. This is usually delayed to avoid repeated decommit/commit calls. +// Note: assumes we (still) own the area as we may purge immediately +static void mi_arena_schedule_purge(mi_arena_t* arena, size_t slice_index, size_t slice_count) { + const long delay = mi_arena_purge_delay(); + if (arena->memid.is_pinned || delay < 0 || _mi_preloading()) return; // is purging allowed at all? + + mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); + if (delay == 0 || _mi_size_of_slices(slice_count) >= mi_option_get_size(mi_option_arena_purge_immediate_size)) { + // purge directly + mi_arena_purge(arena, slice_index, slice_count); + } + else { + // schedule purge + const mi_msecs_t expire = _mi_clock_now() + delay; + mi_msecs_t expire0 = 0; + if (mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &expire0, expire)) { + // expiration was not yet set + // maybe set the global arenas expire as well (if it wasn't set already) + mi_assert_internal(expire0==0); + mi_atomic_casi64_strong_acq_rel(&arena->subproc->purge_expire, &expire0, expire); + } + else { + // already an expiration was set + } + mi_bitmap_setN(arena->slices_purge, slice_index, slice_count, NULL); + } +} + +typedef struct mi_purge_visit_info_s { + mi_msecs_t now; + mi_msecs_t delay; + bool all_purged; + bool any_purged; +} mi_purge_visit_info_t; + +static bool mi_arena_try_purge_range(mi_arena_t* arena, size_t slice_index, size_t slice_count) { + mi_assert(slice_count < MI_BCHUNK_BITS); + if (mi_bbitmap_try_clearNC(arena->slices_free, slice_index, slice_count)) { + // purge + bool decommitted = mi_arena_purge(arena, slice_index, slice_count); MI_UNUSED(decommitted); + mi_assert_internal(!decommitted || mi_bitmap_is_clearN(arena->slices_committed, slice_index, slice_count)); + // and reset the free range + mi_bbitmap_setN(arena->slices_free, slice_index, slice_count); + return true; + } + else { + // was allocated again already + return false; + } +} + +static bool mi_arena_try_purge_visitor(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) { + mi_purge_visit_info_t* vinfo = (mi_purge_visit_info_t*)arg; + // try to purge: first claim the free blocks + if (mi_arena_try_purge_range(arena, slice_index, slice_count)) { + vinfo->any_purged = true; + vinfo->all_purged = true; + } + else if (slice_count > 1) + { + // failed to claim the full range, try per slice instead + for (size_t i = 0; i < slice_count; i++) { + const bool purged = mi_arena_try_purge_range(arena, slice_index + i, 1); + vinfo->any_purged = vinfo->any_purged || purged; + vinfo->all_purged = vinfo->all_purged && purged; + } + } + // don't clear the purge bits as that is done atomically be the _bitmap_forall_set_ranges + // mi_bitmap_clearN(arena->slices_purge, slice_index, slice_count); + return true; // continue +} + +// returns +// -1 = nothing was purged +// 0 = nothing was purged yet because have not yet reached the expire time +// 1 = some pages in the arena were purged +static int mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force) +{ + // check pre-conditions + if (arena->memid.is_pinned) return -1; + + // expired yet? + mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire); + if (expire==0) return -1; + if (!force && expire > now) return 0; + + // reset expire + mi_atomic_storei64_release(&arena->purge_expire, (mi_msecs_t)0); + mi_subproc_stat_counter_increase(arena->subproc, arena_purges, 1); + + // go through all purge info's (with max MI_BFIELD_BITS ranges at a time) + // this also clears those ranges atomically (so any newly freed blocks will get purged next + // time around) + mi_purge_visit_info_t vinfo = { now, mi_arena_purge_delay(), true /*all?*/, false /*any?*/}; + + // we purge by at least `minslices` to not fragment transparent huge pages for example + const size_t minslices = _mi_slice_count_of_size(_mi_os_minimal_purge_size()); + _mi_bitmap_forall_setc_rangesn(arena->slices_purge, minslices, &mi_arena_try_purge_visitor, arena, &vinfo); + + return (vinfo.any_purged ? 1 : -1); +} + + +static void mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq) +{ + // try purge can be called often so try to only run when needed + const long delay = mi_arena_purge_delay(); + if (_mi_preloading() || delay <= 0) return; // nothing will be scheduled + + // check if any arena needs purging? + const mi_msecs_t now = _mi_clock_now(); + const mi_msecs_t arenas_expire = mi_atomic_loadi64_acquire(&subproc->purge_expire); + if (!visit_all && !force && (arenas_expire == 0 || arenas_expire > now)) return; + + const size_t max_arena = mi_arenas_get_count(subproc); + if (max_arena == 0) return; + + // allow only one thread to purge at a time (todo: allow concurrent purging?) + static mi_atomic_guard_t purge_guard; + mi_atomic_guard(&purge_guard) + { + // increase global expire: at most one purge per delay cycle + if (arenas_expire > now) { mi_atomic_storei64_release(&subproc->purge_expire, now + (delay/10)); } + const size_t arena_start = tseq % max_arena; + size_t max_purge_count = (visit_all ? max_arena : (max_arena/4)+1); + bool all_visited = true; + bool any_purged = false; + for (size_t _i = 0; _i < max_arena; _i++) { + size_t i = _i + arena_start; + if (i >= max_arena) { i -= max_arena; } + mi_arena_t* arena = mi_arena_from_index(subproc,i); + if (arena != NULL) { + const int purged = mi_arena_try_purge(arena, now, force); + if (purged >= 0) { // purged, or arena expire is not yet reached + any_purged = true; + if (purged >= 1) { // purged + if (max_purge_count <= 1) { + all_visited = false; + break; + } + max_purge_count--; + } + } + } + } + if (all_visited && !any_purged) { + mi_atomic_storei64_release(&subproc->purge_expire, (mi_msecs_t)0); + } + } +} + +void mi_arenas_purge(void) mi_attr_noexcept { + mi_arenas_try_purge(true, true, _mi_subproc_main(), 0); +} diff --git a/system/lib/mimalloc/src/arena/arena-page.c b/system/lib/mimalloc/src/arena/arena-page.c new file mode 100644 index 0000000000000..8db26247b9544 --- /dev/null +++ b/system/lib/mimalloc/src/arena/arena-page.c @@ -0,0 +1,1374 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#include "mimalloc.h" +#include "mimalloc/prim/prim-tls.h" +#include "mimalloc/util/stats.h" +#include "mimalloc/alloc/page.h" +#include "mimalloc/arena/arena-alloc.h" +#include "mimalloc/profile/sample-guarded.h" // _mi_page_unguard_all +#include "mimalloc/arena/arena-page.h" +#include "arena.h" + +/* ----------------------------------------------------------- + Util +----------------------------------------------------------- */ + +#if MI_DEBUG > 1 +static bool mi_heap_has_page(mi_heap_t* heap, mi_arena_t* arena, mi_page_t* page) { + mi_assert(arena->arena_idx < MI_MAX_ARENAS); + mi_arena_pages_t* arena_pages = heap->arena_pages[arena->arena_idx]; + return (page->memid.memkind == MI_MEM_ARENA && + page->memid.mem.arena.arena == arena && + arena_pages != NULL && + mi_bitmap_is_setN(arena_pages->pages, page->memid.mem.arena.slice_index, 1)); +} +#endif + +mi_page_t* mi_arena_page_at_slice(mi_arena_t* arena, size_t slice_index) { + mi_assert_internal(slice_index < arena->slice_count); + #if MI_PAGE_META_IS_ALIGNED + mi_page_t* const page = mi_aligned_ptr_page(mi_arena_slice_start(arena,slice_index)); // todo: optimize? + return page; + #else + if (arena->pages_meta != NULL) { + mi_page_t* const page = &arena->pages_meta[slice_index]; + #if MI_PAGE_META_SMALL_IS_ALIGNED + // pages with small blocks still have the page at the start of the slice (and set the `block_size` in pages_meta to 0) + if (page->block_size>0) return page; + #else + return page; + #endif + } + // fall through (for MI_PAGE_META_SMALL_IS_ALIGNED) + return (mi_page_t*)mi_arena_slice_start(arena,slice_index); + #endif +} + +static size_t mi_page_full_size(mi_page_t* page) { + if (page->memid.memkind == MI_MEM_ARENA) { + return page->memid.mem.arena.slice_count * MI_ARENA_SLICE_SIZE; + } + else if (mi_memid_is_os(page->memid) || page->memid.memkind == MI_MEM_EXTERNAL) { + mi_assert_internal((uint8_t*)page->memid.mem.os.base <= (uint8_t*)page); + const ptrdiff_t presize = (uint8_t*)page - (uint8_t*)page->memid.mem.os.base; + mi_assert_internal((ptrdiff_t)page->memid.mem.os.size >= presize); + return (presize > (ptrdiff_t)page->memid.mem.os.size ? 0 : page->memid.mem.os.size - presize); + } + else { + return 0; + } +} + +/* ----------------------------------------------------------- + Arena pages: these bitmaps track abandoned pages per size + bin, and the overall pages belonging to a particular heap. +----------------------------------------------------------- */ + +static mi_arena_pages_t* mi_heap_arena_pages(mi_heap_t* heap, mi_arena_t* arena) { + mi_assert_internal(arena!=NULL); + mi_assert_internal(heap!=NULL); + mi_assert(arena->arena_idx < MI_MAX_ARENAS); + return mi_atomic_load_ptr_acquire(mi_arena_pages_t, &heap->arena_pages[arena->arena_idx]); +} + + +static size_t mi_arena_pages_size(size_t slice_count, size_t* bitmap_base) { + if (slice_count == 0) slice_count = MI_BCHUNK_BITS; + mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0); + const size_t base_size = mi_align_up(sizeof(mi_arena_pages_t), MI_BCHUNK_SIZE); + const size_t bitmaps_count = 1 + MI_ARENA_BIN_COUNT; // pages, and abandoned + const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL); + const size_t size = base_size + bitmaps_size; + if (bitmap_base != NULL) *bitmap_base = base_size; + return size; +} + +// allocate initial arena_pages from the main heap +static mi_arena_pages_t* mi_arena_pages_alloc(mi_arena_t* arena) { + const size_t slice_count = arena->slice_count; + size_t bitmap_base = 0; + const size_t size = mi_arena_pages_size(slice_count, &bitmap_base); + mi_arena_pages_t* arena_pages = (mi_arena_pages_t*)mi_heap_zalloc_aligned(arena->subproc->heap_main, size, MI_BCHUNK_SIZE); + if (arena_pages==NULL) return NULL; + uint8_t* base = (uint8_t*)arena_pages + bitmap_base; + mi_assert_internal(mi_is_aligned(base, MI_BCHUNK_SIZE)); + arena_pages->pages = mi_arena_bitmap_init(slice_count, &base); + for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) { + arena_pages->pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base); + } + return arena_pages; +} + +static mi_arena_t* mi_page_arena_pages(mi_page_t* page, size_t* slice_index, size_t* slice_count, mi_arena_pages_t** parena_pages) { + // todo: maybe store the arena* directly in the page? + mi_assert_internal(mi_page_is_owned(page)); + mi_arena_t* const arena = mi_arena_from_memid(page->memid, slice_index, slice_count); + mi_assert_internal(arena != NULL); + if (parena_pages != NULL) { + mi_heap_t* heap = mi_page_heap(page); + mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(heap, arena); + mi_assert_internal(arena_pages != NULL); + mi_assert_internal(slice_index==NULL || mi_bitmap_is_set(arena_pages->pages, *slice_index)); + *parena_pages = arena_pages; + } + return arena; +} + +static mi_arena_pages_t* mi_heap_ensure_arena_pages(mi_heap_t* heap, mi_arena_t* arena) { + mi_assert_internal(arena!=NULL); + mi_assert_internal(heap!=NULL); + mi_assert(arena->arena_idx < MI_MAX_ARENAS); + mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena); + if (arena_pages==NULL) { + mi_lock(&heap->arena_pages_lock) { + arena_pages = mi_atomic_load_ptr_acquire(mi_arena_pages_t, &heap->arena_pages[arena->arena_idx]); + if (arena_pages == NULL) { // still NULL? + if (mi_is_heap_main(heap)) { + // the page info for the main heap is always allocated as part of an arena + arena_pages = &arena->pages_main; + } + else { + // always allocate the arena pages info from the main heap + // todo: allocate into the current arena? + arena_pages = mi_arena_pages_alloc(arena); + } + mi_atomic_store_ptr_release(mi_arena_pages_t, &heap->arena_pages[arena->arena_idx], arena_pages); + } + } + } + if (mi_is_heap_main(heap)) { mi_assert(arena_pages != NULL); } // can never fail + return arena_pages; +} + + +/* ----------------------------------------------------------- + Claim abandoned pages +----------------------------------------------------------- */ + +static bool mi_arena_try_claim_abandoned(size_t slice_index, mi_arena_t* arena, bool* keep_abandoned) { + // found an abandoned page of the right size + mi_page_t* const page = mi_arena_page_at_slice(arena, slice_index); + // can we claim ownership? + if (!mi_page_claim_ownership(page)) { + // there was a concurrent free that reclaims this page .. + // we need to keep it in the abandoned map as the free will call `mi_arena_page_unabandon`, + // and wait for readers (us!) to finish. This is why it is very important to set the abandoned + // bit again (or otherwise the unabandon will never stop waiting). + *keep_abandoned = true; + return false; + } + else { + // yes, we can reclaim it, keep the abandoned map entry clear + *keep_abandoned = false; + return true; + } +} + + +typedef struct mi_arena_abandoned_visit_info_s { + mi_theap_t* theap; + size_t slice_count; + size_t block_size; + size_t bin; +} mi_arena_abandoned_visit_info_t; + +static bool mi_arena_try_find_abandoned_visit(mi_arena_t* arena, const void* arg, void** result) { + const mi_arena_abandoned_visit_info_t* info = (const mi_arena_abandoned_visit_info_t*)arg; + mi_assert_internal(result != NULL); + mi_theap_t* const theap = info->theap; + mi_heap_t* const heap = mi_theap_heap(theap); + const size_t tseq = theap->tld->thread_seq; + mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(heap, arena); + if (arena_pages != NULL) { + size_t slice_index; + mi_bitmap_t* const bitmap = arena_pages->pages_abandoned[info->bin]; + + if (mi_bitmap_try_find_and_claim(bitmap, tseq, &slice_index, &mi_arena_try_claim_abandoned, arena)) { + // found an abandoned page of the right size + // and claimed ownership. + mi_page_t* page = mi_arena_page_at_slice(arena, slice_index); + mi_assert_internal(mi_page_is_owned(page)); + mi_assert_internal(mi_page_is_abandoned(page)); + mi_assert_internal(mi_heap_has_page(heap, arena, page)); + mi_atomic_decrement_relaxed(&heap->abandoned_count[info->bin]); + mi_theap_stat_decrease(theap, pages_abandoned, 1); + mi_theap_stat_counter_increase(theap, pages_reclaim_on_alloc, 1); + + _mi_page_free_collect(page, false); // update `used` count + mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, info->slice_count)); + mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, info->slice_count)); + mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, info->slice_count)); + mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(mi_page_block_size(page) == info->block_size); + mi_assert_internal(!mi_page_is_full(page)); + *result = page; + return false; // break + } + } + return true; // continue +} + +static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t slice_count, size_t block_size) +{ + mi_heap_t* const heap = mi_theap_heap(theap); + const size_t tseq = theap->tld->thread_seq; + mi_arena_t* const req_arena = heap->exclusive_arena; + + const size_t bin = _mi_bin(block_size); + if (bin >= MI_ARENA_BIN_COUNT) { + return NULL; // singleton page size + } + + // any abandoned in our size class? + mi_assert_internal(heap != NULL); + if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) == 0) { + return NULL; + } + + // search arena's + const bool allow_large = true; + const int any_numa = -1; + const bool match_numa = true; + const mi_arena_abandoned_visit_info_t info = { theap, slice_count, block_size, bin }; + void* page = NULL; + mi_forall_suitable_arenas(heap, req_arena, tseq, match_numa, any_numa, allow_large, &mi_arena_try_find_abandoned_visit, &info, &page); + return (mi_page_t*)page; +} + + +/* ----------------------------------------------------------- + Allocate the memory area and page meta data for a fresh page + (`mi_arenas_page_alloc_fresh_area`) +----------------------------------------------------------- */ + +// Offset of the first block when the page meta data is in front of the slices. +static size_t mi_page_meta_in_front_block_start(size_t block_size, bool os_align) { + size_t offset; + #if MI_GUARDED + // in a guarded build, we align pages with blocks a multiple of an OS page size, to the OS page size + // this ensures that all blocks in such pages are OS page size aligned (which is needed for the guard pages) + const size_t os_page_size = _mi_os_page_size(); + mi_assert_internal(MI_PAGE_ALIGN >= os_page_size); + if (!os_align && block_size % os_page_size == 0 && block_size > os_page_size /* at least 2 or more */ ) { + offset = mi_align_up(mi_page_info_size(), os_page_size); + } + else + #endif + if (os_align) { + offset = MI_PAGE_ALIGN; + } + else if (block_size != 0 && mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) { + // naturally align power-of-2 blocks up to MI_PAGE_MAX_START_BLOCK_ALIGN2 size (4KiB) + offset = mi_align_up(mi_page_info_size(), block_size); + } + else if (block_size != 0 && block_size <= MI_SMALL_SIZE_MAX) { + // align small blocks to their size + offset = mi_align_up(mi_page_info_size(), block_size); + } + else if (block_size != 0 && (block_size % MI_PAGE_OSPAGE_BLOCK_ALIGN2) == 0) { + // also align large pages that are a multiple of MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4KiB) + offset = mi_align_up(mi_page_info_size(), MI_PAGE_OSPAGE_BLOCK_ALIGN2); + } + else { + // otherwise start after the info + offset = mi_page_info_size(); + } + return mi_align_up(offset,MI_MAX_ALIGN_SIZE); +} + +// The usable size of a page area (excluding the guard page at the end in secure mode). +static size_t mi_arenas_page_noguard_size(size_t alloc_size) { + #if MI_SECURE>=5 + mi_assert(alloc_size > _mi_os_secure_guard_page_size()); + return alloc_size - _mi_os_secure_guard_page_size(); + #else + return alloc_size; + #endif +} + +// Can we offset the first block to spread cache sets? (see `MI_PAGE_CACHE_OFFSET`) +static bool mi_page_block_can_cache_offset(size_t block_size, bool os_align) { + if (os_align || !mi_block_size_has_cache_offset(block_size)) return false; + #if MI_GUARDED + if (block_size % _mi_os_page_size() == 0) return false; // guarded blocks must stay OS page aligned + #endif + return true; +} + +static size_t mi_page_cache_offset_index(uint8_t* slice_start, size_t slice_count, size_t noffsets) { + // Fibonacci hash of the page index: the top bits spread consecutive pages evenly over the offsets + const size_t page_index = (size_t)((uintptr_t)slice_start / MI_ARENA_SLICE_SIZE) / slice_count; + #if MI_SIZE_BITS >= 64 + const size_t hash = page_index * MI_ZU(0x9E3779B97F4A7C15); + #else + const size_t hash = page_index * MI_ZU(0x9E3779B9); + #endif + return (((hash >> (MI_SIZE_BITS - 16)) * noffsets) >> 16); +} + +static size_t mi_page_small_block_cache_offset(uint8_t* slice_start, size_t slice_count, size_t block_size) { + const size_t unit = mi_max(MI_PAGE_SMALL_CACHE_OFFSET_UNIT, mi_natural_alignment_of_size(block_size)); + if (unit > MI_PAGE_SMALL_CACHE_OFFSET_MAX) return 0; + return mi_page_cache_offset_index(slice_start, slice_count, 1 + MI_PAGE_SMALL_CACHE_OFFSET_MAX/unit) * unit; +} + +// Cache offset for the first block to spread the blocks of different pages over the cache sets (issue #1121). +// The offset fits in the unused space at the end of the page; if there is none, we skip the last block +// if that wastes less than 12.5% of the page (e.g. 64 KiB blocks in a 4 MiB page, but not 512 KiB blocks). +static size_t mi_page_block_cache_offset(uint8_t* slice_start, size_t slice_count, size_t block_size, size_t block_start, bool os_align) { + if (MI_PAGE_CACHE_OFFSET == 0 || os_align) return 0; + if (!mi_block_size_has_cache_offset(block_size)) return mi_page_small_block_cache_offset(slice_start, slice_count, block_size); + if (!mi_page_block_can_cache_offset(block_size, os_align)) return 0; + const size_t page_noguard_size = mi_arenas_page_noguard_size(_mi_size_of_slices(slice_count)); + mi_assert_internal(block_start + block_size <= page_noguard_size); + size_t slack = (page_noguard_size - block_start) % block_size; + if (slack < MI_PAGE_CACHE_OFFSET_UNIT && block_size < page_noguard_size/8) { slack += block_size; } + const size_t noffsets = 1 + (slack < MI_PAGE_CACHE_OFFSET_MAX ? slack : MI_PAGE_CACHE_OFFSET_MAX) / MI_PAGE_CACHE_OFFSET_UNIT; + return mi_page_cache_offset_index(slice_start, slice_count, noffsets) * MI_PAGE_CACHE_OFFSET_UNIT; +} + +// Use the separate (zero initialized) `page_meta` if available and not preferring in front of the slices, +// and otherwise put the page meta data in front of the slices. Also determines the offset of the first block. +static mi_page_t* mi_page_meta_select(uint8_t* slice_start, size_t slice_count, mi_page_t* page_meta, size_t block_size, bool os_align, size_t* block_start) { + mi_assert_internal(page_meta == NULL || page_meta->block_size == 0); + mi_page_t* page; + size_t offset; + if (page_meta == NULL + #if MI_PAGE_META_SMALL_IS_ALIGNED + // small pages keep the page meta data in front of the slices for a fast `mi_free_small` + // (note: it is important that the separate `page_meta->block_size == 0` for `mi_arena_page_at_slice`) + || (!os_align && block_size <= MI_SMALL_MAX_OBJ_SIZE) + #endif + ) { + // put the actual page meta info in front of the slices + // (note: it is not initialized yet since the slices may not be committed yet) + offset = mi_page_meta_in_front_block_start(block_size, os_align); + page = (mi_page_t*)slice_start; + } + else { + // use the separate page meta data + mi_memzero_aligned(page_meta, sizeof(*page_meta)); + offset = 0; + page = page_meta; + } + offset += mi_page_block_cache_offset(slice_start, slice_count, block_size, offset, os_align); + mi_assert_internal(os_align || offset < MI_ARENA_SLICE_SIZE); // so `mi_page_slice_start` can align down the page start + *block_start = offset; + return page; +} + +#if MI_PAGE_META_IS_ALIGNED +// Ensure the aligned page meta data for the slices at `slice_start` is committed. +static bool mi_arena_page_meta_ensure_committed(mi_arena_t* arena, uint8_t* slice_start) { + uint8_t* const meta_slices = (uint8_t*)mi_align_down_ptr(slice_start,MI_PAGE_META_ALIGNMENT); + mi_assert_internal(meta_slices >= mi_arena_start(arena)); + const size_t meta_slice_index = (meta_slices - mi_arena_start(arena)) / MI_ARENA_SLICE_SIZE; + if mi_unlikely(mi_bitmap_is_clear(arena->slices_committed, meta_slice_index)) { + // try to commit all page meta slices now + const size_t meta_slice_count = mi_arena_page_meta_aligned_slice_count(); + // the following assertion does not hold in a concurrent setting.. + // mi_assert_internal(mi_bitmap_is_clearN(arena->slices_committed, meta_slice_index, meta_slice_count)); + const size_t commit_size = meta_slice_count * MI_ARENA_SLICE_SIZE; + if (!_mi_arena_commit(arena->subproc, arena, meta_slices, commit_size, NULL, 0)) { + return false; + } + mi_bitmap_setN(arena->slices_committed, meta_slice_index, meta_slice_count, NULL); + } + return true; +} +#endif + +// Try to allocate slices from free space in the arena's, and determine the page meta data (`*ppage`) and the offset of the first block (`*pblock_start`). +static uint8_t* mi_arenas_page_alloc_from_slices(mi_theap_t* theap, size_t slice_count, size_t block_size, bool commit, bool allow_large, + mi_memid_t* memid, mi_arena_pages_t** parena_pages, mi_page_t** ppage, size_t* pblock_start) { + mi_heap_t* const heap = mi_theap_heap(theap); + mi_tld_t* const tld = theap->tld; + const int numa_node = (heap->numa_node >= 0 ? heap->numa_node : tld->numa_node); + + uint8_t* const start = (uint8_t*)_mi_arenas_try_alloc(heap, slice_count, MI_ARENA_SLICE_ALIGN, commit, allow_large, heap->exclusive_arena, tld->thread_seq, numa_node, memid); + if (start == NULL) return NULL; + mi_arena_t* const arena = memid->mem.arena.arena; + + // ensure the arena pages (and the aligned page meta data) + mi_arena_pages_t* const arena_pages = mi_heap_ensure_arena_pages(heap, arena); + if (arena_pages==NULL + #if MI_PAGE_META_IS_ALIGNED + || !mi_arena_page_meta_ensure_committed(arena, start) + #endif + ) { + _mi_arenas_free(heap->subproc, start, _mi_size_of_slices(slice_count), *memid); // roll back + return NULL; + } + // note: the following assert should hold if we could check it atomically, but in a concurrent setting we may already allocate in slice_count + // mi_assert_internal(mi_bitmap_is_clearN(arena_pages->pages, memid->mem.arena.slice_index, memid->mem.arena.slice_count)); + mi_assert_internal(mi_bitmap_is_clear(arena_pages->pages, memid->mem.arena.slice_index)); + // don't set yet: mi_bitmap_set(arena_pages->pages, memid->mem.arena.slice_index); + *parena_pages = arena_pages; + + // the separate page meta data (if available) + #if MI_PAGE_META_IS_ALIGNED + mi_page_t* const page_meta = mi_aligned_ptr_page0(start); + #else + mi_assert_internal(MI_PAGE_META_IS_SEPARATED!=0 || arena->pages_meta == NULL); + mi_page_t* const page_meta = (arena->pages_meta == NULL ? NULL : &arena->pages_meta[memid->mem.arena.slice_index]); + #endif + *ppage = mi_page_meta_select(start, slice_count, page_meta, block_size, false, pblock_start); + return start; +} + +#if MI_PAGE_META_IS_ALIGNED +// Allocate slices from the OS with the (committed and zero'd) page meta data at the start of the `MI_PAGE_META_ALIGNMENT` aligned area, +// and determine the page meta data (`*ppage`) and the offset of the first block (`*pblock_start`). +static uint8_t* mi_arenas_page_alloc_from_os(mi_heap_t* heap, size_t slice_count, size_t max_page_meta_count, size_t block_size, size_t block_alignment, bool os_align, bool commit, bool allow_large, + mi_memid_t* memid, mi_page_t** ppage, size_t* pblock_start) { + MI_UNUSED(allow_large); + mi_arena_t* const req_arena = heap->exclusive_arena; + const size_t alloc_size = _mi_size_of_slices(slice_count); + if (block_alignment >= MI_PAGE_META_ALIGNMENT) { + // if we allow alignment >= MI_PAGE_META_ALIGNMENT we need to substract 1 from a pointer + // in mi_aligned_ptr_page (and test for (intptr_t)p < 0 instead of NULL). We avoid this by limiting the max alignment. + _mi_warning_message("requested alignment is too large (%zu KiB)\n", block_alignment / MI_KiB); + errno = EINVAL; + return NULL; + // page_offset = MI_PAGE_META_ALIGNMENT; + // os_start = (uint8_t*)_mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, block_alignment, MI_PAGE_META_ALIGNMENT /* align offset */, false /* commit */, false /* allow large */, req_arena, memid); + } + + // offset in the block for the page area + const size_t page_offset = (block_alignment < MI_PAGE_ALIGN ? MI_PAGE_ALIGN : block_alignment); + uint8_t* const os_start = (uint8_t*)_mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, MI_PAGE_META_ALIGNMENT, 0 /* align offset */, false /* commit */, false /* allow large */, req_arena, memid); + if (os_start==NULL) return NULL; + + // commit page info and page + const size_t min_page_count = mi_divide_up(page_offset + MI_ARENA_SLICE_SIZE,MI_ARENA_SLICE_SIZE) + max_page_meta_count; + const size_t min_page_meta = min_page_count * sizeof(mi_page_t); + mi_assert_internal(min_page_meta < page_offset); + bool is_zero; + bool ok = _mi_arena_commit(heap->subproc,req_arena,os_start,min_page_meta,&is_zero,min_page_meta /* don't count in stats? */); + if (ok && commit) { + ok = _mi_arena_commit(heap->subproc,req_arena,os_start + page_offset,alloc_size,NULL,0); + } + if (!ok) { _mi_os_free(heap->subproc,os_start,alloc_size+page_offset,*memid); return NULL; } + if (!is_zero && !memid->initially_zero) { + mi_memzero_aligned(os_start,min_page_meta); + } + uint8_t* const start = os_start + page_offset; + mi_assert_internal(mi_is_aligned(start,block_alignment)); + mi_assert_internal(mi_is_aligned(os_start,MI_PAGE_META_ALIGNMENT)); + mi_assert_internal(mi_align_down_ptr(start-1,MI_PAGE_META_ALIGNMENT) == os_start); + mi_assert_internal((uint8_t*)mi_aligned_ptr_page0(start) < os_start + min_page_meta); + memid->initially_committed = true; // so we don't commit again + *ppage = mi_page_meta_select(start, slice_count, mi_aligned_ptr_page0(start), block_size, os_align, pblock_start); + return start; +} +#else +// Allocate slices from the OS with the page meta data in front of the slices, +// and determine the page meta data (`*ppage`) and the offset of the first block (`*pblock_start`). +static uint8_t* mi_arenas_page_alloc_from_os(mi_heap_t* heap, size_t slice_count, size_t max_page_meta_count, size_t block_size, size_t block_alignment, bool os_align, bool commit, bool allow_large, + mi_memid_t* memid, mi_page_t** ppage, size_t* pblock_start) { + MI_UNUSED(max_page_meta_count); + const size_t page_alignment = MI_ARENA_SLICE_ALIGN; + const size_t alloc_size = _mi_size_of_slices(slice_count); + uint8_t* start; + if (os_align) { + // note: slice_count already includes the page meta data (of MI_PAGE_ALIGN size) in front of the aligned block + start = (uint8_t*)_mi_arena_os_alloc_aligned(heap->subproc, alloc_size, block_alignment, page_alignment /* align offset */, commit, allow_large, heap->exclusive_arena, memid); + mi_assert_internal(start==NULL || mi_is_aligned(start + page_alignment, block_alignment)); + } + else { + start = (uint8_t*)_mi_arena_os_alloc_aligned(heap->subproc, alloc_size, page_alignment, 0 /* align offset */, commit, allow_large, heap->exclusive_arena, memid); + } + if (start == NULL) return NULL; + // page meta data is always placed in front of the slices + *ppage = mi_page_meta_select(start, slice_count, NULL, block_size, os_align, pblock_start); + return start; +} +#endif + +// Allocate the slices for a fresh page and determine its page meta data (`*ppage`) and the offset of the first block (`*pblock_start`). +static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_count, size_t max_page_meta_count, size_t block_size, size_t block_alignment, bool os_align, bool commit, + mi_memid_t* memid, mi_arena_pages_t** parena_pages, mi_page_t** ppage, size_t* pblock_start ) { + mi_assert_internal(parena_pages!=NULL && ppage!=NULL && pblock_start!=NULL); + *parena_pages = NULL; + *memid = _mi_memid_none(); + const bool allow_large = (MI_SECURE < 5); // 5 = guard page at end of each arena page + mi_heap_t* const heap = mi_theap_heap(theap); + + // try to allocate from free space in arena's + uint8_t* slice_start = NULL; + if (!mi_option_is_enabled(mi_option_disallow_arena_alloc) && // allowed to allocate from arena's? + !os_align && // not large alignment + slice_count <= mi_arena_max_object_size()/MI_ARENA_SLICE_SIZE) // and not too large + { + slice_start = mi_arenas_page_alloc_from_slices(theap, slice_count, block_size, commit, allow_large, memid, parena_pages, ppage, pblock_start); + } + + // otherwise fall back to the OS + if (slice_start == NULL) { + slice_start = mi_arenas_page_alloc_from_os(heap, slice_count, max_page_meta_count, block_size, block_alignment, os_align, commit, allow_large, memid, ppage, pblock_start); + if (slice_start == NULL) return NULL; + mi_heap_stat_increase(heap,pages_os_allocated,1); + } + mi_assert_internal(mi_is_aligned(slice_start, MI_PAGE_ALIGN)); + return slice_start; +} + + +/* ----------------------------------------------------------- + Allocate and initialize a fresh page (`mi_arenas_page_alloc_fresh`) +----------------------------------------------------------- */ + +// Free a page without modifying page_bin stats +static void mi_arenas_page_free_prim(mi_page_t* page); + +// Commit the start of the page up to and including the first block if the slices are not yet committed. +// Sets `*commit_size` to the committed size (or 0 if the slices were already committed). +static bool mi_arenas_page_commit_first_block(mi_theap_t* theap, mi_memid_t memid, uint8_t* slice_start, size_t block_start, size_t block_size, size_t page_noguard_size, size_t* commit_size) { + *commit_size = 0; + if (memid.initially_committed) return true; + size_t csize = mi_align_up(block_start + block_size, mi_page_min_commit_size()); + if (csize > page_noguard_size) { csize = page_noguard_size; } + bool is_zero = false; + if mi_unlikely(!_mi_arena_commit(mi_theap_subproc(theap), mi_memid_arena(memid), slice_start, csize, &is_zero, 0)) { + return false; + } + *commit_size = csize; + return true; +} + +// Prepare the page memory: zero the page meta data if it is in front of the slices, set the guard page, etc. +static void mi_arenas_page_init_memory(mi_theap_t* theap, mi_page_t* page, uint8_t* slice_start, size_t slice_count, size_t block_start, size_t page_noguard_size, mi_memid_t* memid) +{ + MI_UNUSED(theap); MI_UNUSED(slice_count); MI_UNUSED(block_start); MI_UNUSED(page_noguard_size); + + // zero initialize the page meta data + if (!memid->initially_zero && (uint8_t*)page == slice_start) { + mi_memzero_aligned(page, sizeof(*page)); + } + + // set the guard page + #if MI_SECURE>=5 + if (memid->initially_committed) { + _mi_os_secure_guard_page_set_at(mi_theap_subproc(theap), slice_start + page_noguard_size, *memid); + } + #endif + + // claimed free slices: initialize the page partly + if (!memid->initially_zero && memid->initially_committed) { + mi_track_mem_undefined(slice_start, slice_count * MI_ARENA_SLICE_SIZE); + } + else if (memid->initially_committed) { + mi_track_mem_defined(slice_start, slice_count * MI_ARENA_SLICE_SIZE); + } + #if MI_DEBUG > 1 + if (memid->initially_zero && memid->initially_committed) { + if (!mi_mem_is_zero(slice_start, page_noguard_size)) { + _mi_error_message(EFAULT, "internal error: page memory was not zero initialized.\n"); + memid->initially_zero = false; + if (block_start > 0) { mi_memzero_aligned(page, sizeof(*page)); } + } + } + #endif +} + +// Initialize the fields of the page meta data. +static void mi_arenas_page_init_meta(mi_theap_t* theap, mi_page_t* page, uint8_t* slice_start, size_t block_start, size_t block_size, bool os_align, + size_t page_noguard_size, size_t commit_size, mi_memid_t memid) +{ + const size_t reserved = (os_align ? 1 : (page_noguard_size - block_start) / block_size); + mi_assert_internal(reserved > 0 && reserved <= UINT16_MAX); + + // initialize the page start + uint8_t* const start = slice_start + block_start; + mi_assert_internal(start > (uint8_t*)page); + page->page_offset = start - (uint8_t*)page; + + // initialize page meta-data + page->reserved = (uint16_t)reserved; + page->block_size = block_size; + page->memid = memid; + page->free_is_zero = memid.initially_zero; + page->slice_pcommitted = (uint16_t)(commit_size / _mi_os_page_size()); + + page->heap = mi_theap_heap(theap); + mi_page_set_theap(page,theap); + // mi_assert_internal(mi_page_theap(page) == _mi_heap_theap_peek(page->heap)) +} + +#if MI_PAGE_META_IS_ALIGNED +// Point the aligned page meta entries of the slices to the actual page meta data. +static void mi_arenas_page_link_aligned_meta(mi_page_t* page, uint8_t* slice_start, size_t slice_count, size_t max_page_meta_count) +{ + mi_page_t* const page_meta = mi_aligned_ptr_page0(slice_start); + mi_atomic_store_ptr_release(mi_page_t,&page_meta->self,page); + if (slice_count > 1) { + // at least two for large singleton blocks as guard pages can have a large offset beyond a single slice + for(size_t i = 1; i < max_page_meta_count; i++) { + mi_assert_internal(page_meta[i].block_size == 0); + mi_atomic_store_ptr_release(mi_page_t,&page_meta[i].self,page); + } + } + #if MI_DEBUG>1 + mi_page_t* pstart = mi_aligned_ptr_page0(slice_start); + mi_assert_internal(mi_atomic_load_ptr_acquire(mi_page_t,&pstart->self)==page); + if (page->reserved>1) { + mi_page_t* pend = mi_aligned_ptr_page0(slice_start + (slice_count*MI_ARENA_SLICE_SIZE) - 1); + mi_assert_internal(mi_atomic_load_ptr_acquire(mi_page_t,&pend->self)==page); + } + #endif +} +#endif + +// Register a fresh page in its arena pages and the page map, and claim ownership. +// On failure the page is freed. +static bool mi_arenas_page_register(mi_theap_t* theap, mi_page_t* page, mi_arena_pages_t* arena_pages) +{ + // register in the arena_pages + if (arena_pages!=NULL) { + mi_assert_internal(page->memid.memkind == MI_MEM_ARENA); + mi_bitmap_set(arena_pages->pages, page->memid.mem.arena.slice_index); + } + + // and own it + mi_page_claim_ownership(page); + + // register in the page map + if mi_unlikely(!_mi_page_map_register(page)) { + mi_arenas_page_free_prim(page); + return false; + } + + // stats + mi_theap_stat_increase(theap, pages, 1); + mi_theap_stat_increase(theap, page_bins[_mi_page_stats_bin(page)], 1); + return true; +} + +// Allocate a fresh page +static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool commit) +{ + const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN); + const bool singleton = (os_align || block_size > MI_LARGE_MAX_OBJ_SIZE); + const size_t max_page_meta_count = (singleton && slice_count > 2 ? 2 : slice_count); + + // allocate the slices and determine the page meta data + mi_memid_t memid = _mi_memid_none(); + mi_arena_pages_t* arena_pages = NULL; + mi_page_t* page = NULL; + size_t block_start = 0; + uint8_t* const slice_start = mi_arenas_page_alloc_fresh_area(theap,slice_count,max_page_meta_count,block_size,block_alignment,os_align,commit,&memid,&arena_pages,&page,&block_start); + if (!slice_start) return NULL; + mi_assert_internal(block_size < MI_MAX_ALIGN_SIZE || block_start % MI_MAX_ALIGN_SIZE == 0); + // all blocks must be naturally aligned (as relied upon by `mi_malloc_aligned`) + mi_assert_internal(block_size % mi_natural_alignment_of_size(block_size) == 0); + mi_assert_internal(mi_is_aligned(slice_start + block_start, mi_natural_alignment_of_size(block_size))); + + // commit the first block (if needed) + const size_t alloc_size = _mi_size_of_slices(slice_count); + const size_t page_noguard_size = mi_arenas_page_noguard_size(alloc_size); + size_t commit_size = 0; + if (!mi_arenas_page_commit_first_block(theap, memid, slice_start, block_start, block_size, page_noguard_size, &commit_size)) { + _mi_arenas_free(mi_theap_subproc(theap), slice_start, alloc_size, memid); + return NULL; + } + mi_assert_internal((commit && commit_size==0) || (!commit && (commit_size <= UINT16_MAX * _mi_os_page_size()))); + + // initialize the page memory and meta data + mi_arenas_page_init_memory(theap, page, slice_start, slice_count, block_start, page_noguard_size, &memid); + mi_arenas_page_init_meta(theap, page, slice_start, block_start, block_size, os_align, page_noguard_size, commit_size, memid); + #if MI_PAGE_META_IS_ALIGNED + mi_arenas_page_link_aligned_meta(page, slice_start, slice_count, max_page_meta_count); + #endif + mi_assert_internal(page->free==NULL); + mi_assert_internal(((uint8_t*)page != slice_start) == mi_page_meta_is_separated(page)); + mi_assert_internal(mi_page_slice_start(page) == slice_start); + mi_assert_internal(mi_page_size(page) <= page_noguard_size); + + // now we can register the page (and use `mi_arenas_page_free_prim` on error) + if (!mi_arenas_page_register(theap, page, arena_pages)) return NULL; + + mi_assert_internal(mi_is_aligned(mi_page_slice_start(page),MI_PAGE_ALIGN)); + mi_assert_internal(mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(mi_page_block_size(page) == block_size); + #if MI_PAGE_META_SMALL_IS_ALIGNED + // `mi_free_small` finds the page meta data by aligning a block pointer down to `MI_SMALL_PAGE_SIZE` + if (mi_page_is_small(page)) { + mi_assert_internal((uint8_t*)page == slice_start && mi_is_aligned(page, MI_SMALL_PAGE_SIZE)); + mi_assert_internal(mi_page_start(page) + mi_page_size(page) <= (uint8_t*)page + MI_SMALL_PAGE_SIZE); + } + #endif + // mi_assert_internal(mi_page_is_abandoned(page)); + mi_assert_internal(mi_page_is_owned(page)); + return page; +} + + + +/* ----------------------------------------------------------- + Allocate page (`_mi_arenas_page_alloc`) +----------------------------------------------------------- */ + +// Allocate a regular small/medium/large page. +static mi_page_t* mi_arenas_page_regular_alloc(mi_theap_t* theap, size_t slice_count, size_t block_size) +{ + // 1. look for an abandoned page + mi_page_t* page = mi_arenas_page_try_find_abandoned(theap, slice_count, block_size); + if (page != NULL) { + return page; // return as abandoned + } + + // 2. find a free block, potentially allocating a new arena + const long commit_on_demand = mi_option_get(mi_option_page_commit_on_demand); + const bool commit = (mi_page_min_commit_size() >= slice_count * MI_ARENA_SLICE_SIZE || // always commit small pages + (slice_count >= _mi_slice_count_of_size(UINT16_MAX * _mi_os_page_size())) || // always commit pages too large to hold a 32-bit slice_committed + (commit_on_demand == 2 && _mi_os_has_overcommit()) || (commit_on_demand == 0)); + page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, 1, commit); + if (page == NULL) return NULL; + + mi_assert_internal(page->memid.memkind != MI_MEM_ARENA || page->memid.mem.arena.slice_count == slice_count); + if (!_mi_page_init(theap, page)) { + _mi_arenas_page_free(page,theap); + return NULL; + } + + return page; +} + +// Allocate a page containing one block (very large, or with large alignment) +static mi_page_t* mi_arenas_page_singleton_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment) +{ + const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN); + #if MI_PAGE_META_IS_ALIGNED + const size_t info_size = 0; + #else + const size_t info_size = (os_align ? MI_PAGE_ALIGN : mi_page_info_size()); + #endif + // room for the cache offset of the block (committed but not touched) + const size_t cache_offset_size = (mi_page_block_can_cache_offset(block_size, os_align) ? MI_PAGE_CACHE_OFFSET_MAX : 0); + #if MI_SECURE < 2 + const size_t slice_count = _mi_slice_count_of_size(info_size + cache_offset_size + block_size); + #else + const size_t slice_count = _mi_slice_count_of_size(mi_align_up(info_size + cache_offset_size + block_size, _mi_os_secure_guard_page_size()) + _mi_os_secure_guard_page_size()); + #endif + + mi_page_t* page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, block_alignment, true /* commit singletons always */); + if (page == NULL) return NULL; + + mi_assert(page->reserved == 1); + if (!_mi_page_init(theap, page)) { + _mi_arenas_page_free(page,theap); + return NULL; + } + + return page; +} + + +mi_page_t* _mi_arenas_page_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment) { + mi_page_t* page; + // semi static assert: ensure that all non-singleton block size bins are covered. + mi_assert(_mi_bin(MI_LARGE_MAX_OBJ_SIZE) < MI_ARENA_BIN_COUNT); + if mi_unlikely(block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) { + mi_assert_internal(mi_is_power_of_two(block_alignment)); + page = mi_arenas_page_singleton_alloc(theap, block_size, block_alignment); + } + else if (block_size <= MI_SMALL_MAX_OBJ_SIZE) { + page = mi_arenas_page_regular_alloc(theap, _mi_slice_count_of_size(MI_SMALL_PAGE_SIZE), block_size); + } + else if (mi_block_size_is_medium(block_size)) { + page = mi_arenas_page_regular_alloc(theap, _mi_slice_count_of_size(MI_MEDIUM_PAGE_SIZE), block_size); + } + #if MI_ENABLE_LARGE_PAGES + else if (block_size <= MI_LARGE_MAX_OBJ_SIZE) { + page = mi_arenas_page_regular_alloc(theap, _mi_slice_count_of_size(MI_LARGE_PAGE_SIZE), block_size); + } + #endif + else { + page = mi_arenas_page_singleton_alloc(theap, block_size, block_alignment); + } + if mi_unlikely(page == NULL) { + return NULL; + } + // mi_assert_internal(page == NULL || _mi_page_segment(page)->subproc == tld->subproc); + mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(block_alignment <= MI_PAGE_MAX_OVERALLOC_ALIGN || mi_is_aligned(mi_page_start(page), block_alignment)); + + return page; +} + + +/* ----------------------------------------------------------- + Free a page +----------------------------------------------------------- */ + +static void mi_arenas_page_free_prim(mi_page_t* page) { + mi_assert_internal(_mi_page_qsbr_is_clear(page)); + mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(mi_page_is_owned(page)); + mi_assert_internal(mi_page_all_free(page)); + mi_assert_internal(page->next==NULL && page->prev==NULL); + + #if MI_DEBUG>1 + if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) { + size_t bin = _mi_bin(mi_page_block_size(page)); + size_t slice_index; + size_t slice_count; + mi_arena_pages_t* arena_pages = NULL; + mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); + mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); + mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + mi_assert_internal(bin >= MI_ARENA_BIN_COUNT || mi_bitmap_is_clearN(arena_pages->pages_abandoned[bin], slice_index, 1)); + mi_assert_internal(mi_bitmap_is_setN(arena_pages->pages, slice_index, 1)); + // note: we cannot check for `!mi_page_is_abandoned_and_mapped` since that may + // be (temporarily) not true if the free happens while trying to reclaim + // see `mi_arena_try_claim_abandoned` + } + #endif + + // unregister page + _mi_page_map_unregister(page); + + // recommit guard page at the end? + // we must do this since we may later allocate large spans over this page and cannot have a guard page in between + #if MI_SECURE >= 5 + if (!page->memid.is_pinned) { + _mi_os_secure_guard_page_reset_before(mi_page_subproc(page), mi_page_slice_start(page) + mi_page_full_size(page), page->memid); + } + #endif + + // and free + if (page->memid.memkind == MI_MEM_ARENA) { + mi_arena_pages_t* arena_pages; + size_t slice_index; + size_t slice_count; MI_UNUSED(slice_count); + mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); + mi_assert_internal(arena_pages!=NULL); + mi_assert_internal(arena->subproc == mi_page_subproc(page)); + mi_bitmap_clear(arena_pages->pages, slice_index); + const size_t slice_committed = mi_page_slice_committed(page); + if (slice_committed > 0) { + // if committed on-demand, set the commit bits to account commit properly + mi_assert_internal(mi_page_full_size(page) >= slice_committed); + const size_t total_slices = slice_committed / MI_ARENA_SLICE_SIZE; // conservative + //mi_assert_internal(mi_bitmap_is_clearN(arena->slices_committed, slice_index, total_slices)); + mi_assert_internal(slice_count >= total_slices); + if (total_slices > 0) { + mi_bitmap_setN(arena->slices_committed, slice_index, total_slices, NULL); + } + // any left over? + const size_t extra = slice_committed % MI_ARENA_SLICE_SIZE; + if (extra > 0) { + // pretend it was decommitted already + mi_subproc_stat_decrease(arena->subproc, committed, extra); + } + } + else { + mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + } + } + else { + mi_heap_stat_decrease(page->heap, pages_os_allocated, 1); + } + if (mi_page_meta_is_separated(page)) { page->block_size = 0; } // for assertion checking + _mi_arenas_free( mi_page_subproc(page), mi_page_slice_start(page), mi_page_full_size(page), page->memid); +} + +void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) { + mi_assert_internal(_mi_page_qsbr_is_clear(page)); + mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(mi_page_is_owned(page)); + mi_assert_internal(mi_page_all_free(page)); + mi_assert_internal(mi_page_is_abandoned(page)); + mi_assert_internal(page->next==NULL && page->prev==NULL); + mi_assert_internal(mi_theap_matches_thread(current_theapx)); + + mi_heap_t* const heap = mi_page_heap(page); + mi_theapx_stat_decrease(heap, current_theapx, page_bins[_mi_page_stats_bin(page)], 1); + mi_theapx_stat_decrease(heap, current_theapx, pages, 1); + _mi_page_free_collect(page,false); // update used count for cross-thread free's + _mi_page_update_stats(page); // and update the stats + mi_arenas_page_free_prim(page); +} + + +/* ----------------------------------------------------------- + Abandon a page +----------------------------------------------------------- */ + + +// release ownership of a page. This may free the page if all blocks were concurrently +// freed in the meantime. Returns true if the page was freed. +static bool mi_abandoned_page_unown(mi_page_t* page, mi_theap_t* current_theapx) { + mi_assert_internal(mi_page_is_owned(page)); + mi_assert_internal(mi_page_is_abandoned(page)); + mi_assert_internal(mi_theap_matches_thread(current_theapx)); + mi_thread_free_t tf_new; + mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free); + do { + mi_assert_internal(mi_tf_is_owned(tf_old)); + while mi_unlikely(mi_tf_block(tf_old) != NULL) { + _mi_page_free_collect(page, false); // update used + if (mi_page_all_free(page)) { + if (!_mi_page_qsbr_should_delay(page)) { + _mi_page_qsbr_clear(page); + _mi_arenas_page_unabandon(page, current_theapx); + _mi_arenas_page_free(page, current_theapx); + return true; + } + #if MI_QSBR + else if (page->memid.memkind == MI_MEM_ARENA && + !mi_page_is_singleton(page) && !mi_page_is_abandoned_mapped(page)) + { + // if empty and qsbr delayed, we re-abandon to mapped so it can be reused + _mi_arenas_page_unabandon(page, current_theapx); + _mi_arenas_page_abandon(page, current_theapx); // cannot recurse due to `!mi_page_is_abandoned_mapped(page)` + return false; + } + #endif + } + tf_old = mi_atomic_load_relaxed(&page->xthread_free); + } + mi_assert_internal(mi_tf_block(tf_old)==NULL); + tf_new = mi_tf_create(NULL, false); + } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new)); + return false; +} + + +void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theapx) { + mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(mi_page_is_owned(page)); + mi_assert_internal(mi_page_is_abandoned(page)); + mi_assert_internal(MI_QSBR || !mi_page_all_free(page)); + mi_assert_internal(page->next==NULL && page->prev == NULL); + mi_assert_internal(mi_theap_matches_thread(current_theapx)); + // mi_assert_internal(current_theap == _mi_page_associated_theap(page)); + #if MI_QSBR + mi_assert_internal(!page->qsbr_in_queue && page->qsbr_next == NULL && page->qsbr_prev == NULL); + #endif + + // note: somewhat expensive to update here, but might be good as then we attribute + // the current allocations/frees to the current thread/theap. Otherwise it might be + // reclaimed later in another thread/theap and those allocations/frees get attributed there... + _mi_page_update_stats(page); + + // add to abandoned? + mi_heap_t* heap = mi_page_heap(page); + if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page) + #if MI_QSBR + && !mi_page_is_singleton(page) // an arena singleton has size > MI_LARGE_MAX_OBJ_SIZE and bin==MI_BIN_HUGE + #endif + ) { + // make available for allocations + size_t bin = _mi_bin(mi_page_block_size(page)); + mi_assert_internal(bin < MI_ARENA_BIN_COUNT); + if (bin < MI_ARENA_BIN_COUNT) { // paranoia + size_t slice_index; + size_t slice_count; + mi_arena_pages_t* arena_pages = NULL; + mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); MI_UNUSED(arena); + + mi_assert_internal(!mi_page_is_singleton(page)); + mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); + mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count)); + + mi_page_set_abandoned_mapped(page); + const bool was_clear = mi_bitmap_set(arena_pages->pages_abandoned[bin], slice_index); + MI_UNUSED(was_clear); mi_assert_internal(was_clear); + mi_atomic_increment_relaxed(&heap->abandoned_count[bin]); + mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1); + mi_abandoned_page_unown(page, current_theapx); + return; + } + } + // otherwise, + // page is full (or a singleton), or the page is OS/externally allocated + // leave as is; it will be reclaimed when an object is free'd in the page + // but for non-arena pages, add to the subproc list so these can be visited + if (page->memid.memkind != MI_MEM_ARENA) { + mi_lock(&heap->os_abandoned_pages_lock) { + // push in front + page->prev = NULL; + page->next = heap->os_abandoned_pages; + if (page->next != NULL) { page->next->prev = page; } + heap->os_abandoned_pages = page; + } + mi_theapx_stat_increase(heap, current_theapx, pages_os_abandoned, 1); + } + mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1); + mi_abandoned_page_unown(page, current_theapx); +} + + +// this is called from `alloc/free.c:mi_free_try_collect_mt` only. +bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page) { + mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(mi_page_is_owned(page)); + mi_assert_internal(mi_page_is_abandoned(page)); + mi_assert_internal(!mi_page_is_abandoned_mapped(page)); + mi_assert_internal(!mi_page_is_full(page)); + mi_assert_internal(!mi_page_all_free(page)); + mi_assert_internal(!mi_page_is_singleton(page)); + if (mi_page_is_full(page) || mi_page_is_abandoned_mapped(page) || page->memid.memkind != MI_MEM_ARENA) { + return false; + } + else { + // do not use _mi_heap_theap as we may call this during shutdown of threads and don't want to reinitialize the theap + mi_theap_t* const theapx = _mi_page_associated_theap_peek(page); // can be NULL + mi_heap_t* const heap = mi_page_heap(page); + // if (theapx==NULL) return false; + mi_theapx_stat_counter_increase(heap, theapx, pages_reabandon_full, 1); + mi_theapx_stat_adjust_decrease(heap, theapx, pages_abandoned, 1); // adjust as we are not abandoning fresh + _mi_arenas_page_abandon(page, theapx); + return true; + } +} + +// called from `mi_free` if trying to unabandon an abandoned page +void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) { + mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(mi_page_is_owned(page)); + mi_assert_internal(mi_page_is_abandoned(page)); + mi_assert_internal(mi_theap_matches_thread(current_theapx)); + + mi_heap_t* const heap = mi_page_heap(page); + if (mi_page_is_abandoned_mapped(page)) { + mi_assert_internal(page->memid.memkind==MI_MEM_ARENA); + // remove from the abandoned map + const size_t bin = _mi_bin(mi_page_block_size(page)); + mi_assert_internal(bin < MI_ARENA_BIN_COUNT); + size_t slice_index; + size_t slice_count; + mi_arena_pages_t* arena_pages; + mi_arena_t* arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); MI_UNUSED(arena); + + mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); + mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + + // this busy waits until a concurrent reader (from alloc_abandoned) is done + mi_bitmap_clear_once_set(arena->subproc, arena_pages->pages_abandoned[bin], slice_index); + mi_page_clear_abandoned_mapped(page); + mi_atomic_decrement_relaxed(&heap->abandoned_count[bin]); + } + else { + // page is full (or a singleton), page is OS allocated + // if not an arena page, remove from the subproc os pages list + if (page->memid.memkind != MI_MEM_ARENA) { + mi_lock(&heap->os_abandoned_pages_lock) { + if (page->prev != NULL) { page->prev->next = page->next; } + if (page->next != NULL) { page->next->prev = page->prev; } + if (heap->os_abandoned_pages == page) { heap->os_abandoned_pages = page->next; } + page->next = NULL; + page->prev = NULL; + } + mi_theapx_stat_decrease(heap, current_theapx, pages_os_abandoned, 1); + } + } + mi_theapx_stat_decrease(heap, current_theapx, pages_abandoned, 1); +} + + +/* ----------------------------------------------------------- + Visit all pages and blocks in a heap +----------------------------------------------------------- */ + +typedef struct mi_heap_visit_info_s { + mi_heap_t* heap; + mi_block_visit_fun* visitor; + void* arg; + bool visit_blocks; + bool abandoned_only; +} mi_heap_visit_info_t; + +static bool mi_heap_visit_page(mi_page_t* page, const mi_heap_visit_info_t* vinfo) { + mi_heap_area_t area; + #if MI_QSBR + const bool claimed = (mi_page_is_abandoned(page) && mi_page_claim_ownership(page)); + #endif + _mi_page_free_collect(page, true); // collect cross-thread free'd blocks so `area.used` is precise. + #if MI_QSBR + if (claimed && mi_abandoned_page_unown(page, NULL)) return true; + #endif + _mi_heap_area_init(&area, page); + mi_assert_internal(vinfo->heap == mi_page_heap(page)); + if (!vinfo->visitor(vinfo->heap, &area, NULL, area.block_size, vinfo->arg)) { + return false; // break + } + if (vinfo->visit_blocks) { + return _mi_theap_area_visit_blocks(&area, page, vinfo->visitor, vinfo->arg); // continue or break + } + else { + return true; // continue + } +} + +static bool mi_heap_visit_page_at(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) { + MI_UNUSED(slice_count); + const mi_heap_visit_info_t* vinfo = (const mi_heap_visit_info_t*)arg; + mi_page_t* page = mi_arena_page_at_slice(arena, slice_index); + return mi_heap_visit_page(page, vinfo); // continue or break +} + +static bool mi_heap_visit_arena(mi_arena_t* arena, const void* arg, void** result) { + MI_UNUSED(result); + const mi_heap_visit_info_t* vinfo = (const mi_heap_visit_info_t*)arg; + mi_heap_visit_info_t visit_info = *vinfo; + mi_heap_t* const heap = vinfo->heap; + mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena); + if (arena_pages == NULL) return true; // continue + if (vinfo->abandoned_only) { + for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) { + // TODO: if we had a single abandoned page map as well, this can be faster. Maybe for CPython? + if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) > 0) { + if (!_mi_bitmap_forall_set(arena_pages->pages_abandoned[bin], &mi_heap_visit_page_at, arena, &visit_info)) return false; // break + } + } + return true; // continue + } + else { + return _mi_bitmap_forall_set(arena_pages->pages, &mi_heap_visit_page_at, arena, &visit_info); // continue or break + } +} + +// The caller holds heap->theaps_lock to prevent thread-exit abandonment and theap removal. +static bool mi_heap_visit_owned_os_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { + const mi_heap_visit_info_t visit_info = { heap, visitor, arg, visit_blocks, false }; + for (mi_theap_t* theap = heap->theaps; theap != NULL; theap = theap->hnext) { + if (theap->page_count_os == 0) continue; + for (size_t bin = 0; bin <= MI_BIN_FULL; bin++) { + mi_page_t* page = theap->pages[bin].first; + while (page != NULL) { + mi_page_t* const next = page->next; + if (page->memid.memkind != MI_MEM_ARENA && !mi_heap_visit_page(page, &visit_info)) return false; + page = next; + } + } + } + return true; +} + +static bool mi_heap_visit_blocks_locked(mi_heap_t* heap, bool abandoned_only, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { + // visit all pages in a heap + // Other threads must not access this heap; thread-exit abandonment is blocked by the heap lock. + const mi_heap_visit_info_t visit_info = { heap, visitor, arg, visit_blocks, abandoned_only }; + bool ok = mi_forall_arenas(heap, NULL, 0, &mi_heap_visit_arena, &visit_info, NULL); + if (!ok) return false; + + // visit abandoned pages in OS allocated memory + mi_page_t* page = NULL; + mi_lock(&heap->os_abandoned_pages_lock) { + page = heap->os_abandoned_pages; + } + while (ok && page != NULL) { + mi_page_t* next = page->next; // read upfront in case the visitor frees the page + ok = mi_heap_visit_page(page, &visit_info); + page = next; + } + + // visit owned OS allocated blocks in the theaps + if (ok && !abandoned_only) { + ok = mi_heap_visit_owned_os_blocks(heap, visit_blocks, visitor, arg); + } + return ok; +} + +bool _mi_heap_visit_blocks(mi_heap_t* heap, bool abandoned_only, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { + mi_assert(visitor!=NULL); + if (visitor==NULL) return false; + if (heap==NULL) { heap = mi_heap_main(); } + bool ok; + mi_lock(&heap->theaps_lock) { + ok = mi_heap_visit_blocks_locked(heap, abandoned_only, visit_blocks, visitor, arg); + } + return ok; +} + +bool mi_heap_visit_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { + return _mi_heap_visit_blocks(heap, false, visit_blocks, visitor, arg); +} + +bool mi_heap_visit_abandoned_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { + return _mi_heap_visit_blocks(heap, true, visit_blocks, visitor, arg); +} + +#if MI_QSBR +static bool mi_heap_collect_abandoned_page(mi_page_t* page) { + if (!mi_page_is_abandoned(page) || !mi_page_claim_ownership(page)) return true; + _mi_page_free_collect(page, true); + if (mi_page_all_free(page) && !_mi_page_qsbr_should_delay(page)) { + _mi_page_qsbr_clear(page); + _mi_arenas_page_unabandon(page, NULL); + _mi_arenas_page_free(page, NULL); + } + else { + mi_abandoned_page_unown(page, NULL); + } + return true; +} + +static bool mi_heap_collect_abandoned_page_at(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) { + MI_UNUSED(slice_count); MI_UNUSED(arg); + return mi_heap_collect_abandoned_page(mi_arena_page_at_slice(arena, slice_index)); +} + +static bool mi_heap_collect_abandoned_arena(mi_arena_t* arena, const void* arg, void** result) { + MI_UNUSED(result); + mi_heap_t* heap = (mi_heap_t*)arg; + mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena); + if (arena_pages == NULL) return true; + return _mi_bitmap_forall_set(arena_pages->pages, &mi_heap_collect_abandoned_page_at, arena, NULL); +} + +void _mi_heap_collect_abandoned(mi_heap_t* heap) { + mi_assert_internal(heap != NULL); + mi_lock(&heap->theaps_lock) { + // Use the all-pages map: abandoned size-class maps exclude singletons and full pages. + mi_forall_arenas(heap, NULL, 0, &mi_heap_collect_abandoned_arena, heap, NULL); + mi_page_t* page = NULL; + mi_lock(&heap->os_abandoned_pages_lock) { page = heap->os_abandoned_pages; } + while (page != NULL) { + mi_page_t* next = page->next; + mi_heap_collect_abandoned_page(page); + page = next; + } + } +} +#endif + + +typedef struct mi_heap_delete_visit_info_s { + mi_heap_t* heap_target; + mi_theap_t* theap_target; + mi_theap_t* theap; +} mi_heap_delete_visit_info_t; + +static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg) { + MI_UNUSED(block); MI_UNUSED(block_size); MI_UNUSED(heap); + const mi_heap_delete_visit_info_t* info = (const mi_heap_delete_visit_info_t*)arg; + mi_heap_t* heap_target = info->heap_target; + mi_theap_t* const theap = NULL; // info->theap; mi_assert_internal(mi_theap_heap(theap) == heap); + mi_page_t* const page = (mi_page_t*)area->reserved1; + + mi_page_claim_ownership(page); // claim ownership + _mi_page_qsbr_clear(page); // deletion requires an unprotected heap; destruction requires no readers + if (mi_page_is_abandoned(page)) { + _mi_arenas_page_unabandon(page,theap); + } + else { + page->next = page->prev = NULL; // yikes.. better not to try to access this from a thread later on.. + mi_page_set_theap(page,NULL); // set threadid to abandoned + } + mi_assert_internal(mi_page_is_abandoned(page)); + mi_assert_internal(mi_page_is_owned(page)); + + if (mi_page_used(page)==0) { + // free the page + _mi_arenas_page_free(page, theap); + } + else if (heap_target==NULL) { + #if MI_GUARDED + _mi_page_unguard_all(page); // remove potential interior guard pages + #endif + // destroy the page + _mi_page_free_collect(page, false); // collect the thread-free list first while `used` still counts those blocks (as `_mi_arenas_page_free` collects it again) (see #1420) + mi_page_used_reset(page); // note: invariant `|local_free| + |free| == reserved - used` does not hold in this case + _mi_arenas_page_free(page, theap); + } + else { + // move the page to `heap_target` as an abandoned page + // first remove it from the current heap + const size_t sbin = _mi_page_stats_bin(page); + mi_arena_t* arena = NULL; + size_t slice_index = 0; + if (page->memid.memkind == MI_MEM_ARENA) { + size_t slice_count; + mi_arena_pages_t* arena_pages = NULL; + arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); + mi_assert_internal(mi_bitmap_is_set(arena_pages->pages, slice_index)); + mi_bitmap_clear(arena_pages->pages, slice_index); + } + else { + // os allocated + mi_assert_internal(mi_memid_is_os(page->memid) && page->next == NULL); + } + if (theap != NULL) { + mi_theap_stat_decrease(theap, page_bins[sbin], 1); + mi_theap_stat_decrease(theap, pages, 1); + } + else { + mi_heap_stat_decrease((mi_heap_t*)heap, page_bins[_mi_page_stats_bin(page)], 1); + mi_heap_stat_decrease((mi_heap_t*)heap, pages, 1); + } + mi_theap_t* theap_target = info->theap_target; + + // and then add it to the new target heap + if (arena != NULL) { + mi_arena_pages_t* arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena); + if mi_unlikely(arena_pages_target==NULL) { + // if we cannot allocate this, we move it to the main heap instead (which does not require allocation) + heap_target = _mi_arena_heap_main(arena); + theap_target = mi_heap_theap(heap_target); // todo: find through theap_target tld? + arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena); + mi_assert_internal(arena_pages_target!=NULL); + } + mi_assert_internal(mi_bitmap_is_clear(arena_pages_target->pages, slice_index)); + mi_bitmap_set(arena_pages_target->pages, slice_index); + } + page->heap = heap_target; + mi_theap_stat_increase(theap_target, page_bins[sbin], 1); + mi_theap_stat_increase(theap_target, pages, 1); + + // and abandon in the new heap + _mi_arenas_page_abandon(page,theap_target); + } + return true; // continue +} + +static void mi_heap_delete_pages(mi_heap_t* heap, mi_heap_t* heap_target) { + mi_theap_t* const theap_target = (heap_target != NULL ? _mi_heap_theap(heap_target) : NULL); + // mi_theap_t* const theap = _mi_heap_theap(heap); + mi_heap_delete_visit_info_t info = { heap_target, theap_target, NULL }; + _mi_heap_visit_blocks(heap, false, false, &mi_heap_delete_page, &info); + #if MI_DEBUG>1 + // no more arena pages? + for (size_t i = 0; i < MI_MAX_ARENAS; i++) { + mi_arena_pages_t* const arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]); + if (arena_pages!=NULL) { + mi_assert_internal(mi_bitmap_is_all_clear(arena_pages->pages)); + } + } + // nor os abandoned pages? + mi_lock(&heap->os_abandoned_pages_lock) { + + mi_assert_internal(heap->os_abandoned_pages == NULL); + } + // nor arena abandoned pages? + for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) { + mi_assert_internal(mi_atomic_load_relaxed(&heap->abandoned_count[i])==0); + } + #endif +} + +void _mi_heap_move_pages(mi_heap_t* heap_from, mi_heap_t* heap_to) { + if (mi_is_heap_main(heap_from)) return; + if (heap_to==NULL) { heap_to = heap_from->subproc->heap_main; } + mi_heap_delete_pages(heap_from, heap_to); +} + +void _mi_heap_destroy_pages(mi_heap_t* heap_from) { + if (mi_is_heap_main(heap_from)) return; + mi_heap_delete_pages(heap_from, NULL); +} diff --git a/system/lib/mimalloc/src/arena/arena.c b/system/lib/mimalloc/src/arena/arena.c new file mode 100644 index 0000000000000..ea7fb7d750eac --- /dev/null +++ b/system/lib/mimalloc/src/arena/arena.c @@ -0,0 +1,1188 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +/* ---------------------------------------------------------------------------- +"Arenas" are fixed area's of OS memory from which we can allocate +large blocks (>= MI_ARENA_MIN_BLOCK_SIZE, 64KiB). +In contrast to the rest of mimalloc, the arenas are shared between +threads and need to be accessed using atomic operations. + +Arenas are also used to for huge OS page (1GiB) reservations or for reserving +OS memory upfront which can be improve performance or is sometimes needed +on embedded devices. We can also employ this with WASI or `sbrk` systems +to reserve large arenas upfront and be able to reuse the memory more effectively. + +The arena allocation needs to be thread safe and we use an atomic bitmap to allocate. +-----------------------------------------------------------------------------*/ + +#include "mimalloc.h" +#include "mimalloc/util/stats.h" +#include "mimalloc/alloc/page.h" +#include "mimalloc/heap/subproc.h" +#include "mimalloc/arena/arena.h" +#include "arena.h" + +mi_heap_t* _mi_arena_heap_main(const mi_arena_t* arena) { + return _mi_subproc_heap_main(arena->subproc); +} + +/* ----------------------------------------------------------- + arena blocks +----------------------------------------------------------- */ + +// Blocks needed for a given byte size +size_t _mi_slice_count_of_size(size_t size) { + return mi_divide_up(size, MI_ARENA_SLICE_SIZE); +} + +// Byte size of a number of blocks +size_t _mi_size_of_slices(size_t bcount) { + return (bcount * MI_ARENA_SLICE_SIZE); +} + +mi_memid_t _mi_memid_create(mi_memkind_t memkind) { + mi_memid_t memid; + _mi_memzero_var(memid); + memid.memkind = memkind; + return memid; +} + +mi_memid_t _mi_memid_none(void) { + return _mi_memid_create(MI_MEM_NONE); +} + +mi_memid_t _mi_memid_create_os(void* base, size_t size, bool committed, bool is_zero, bool is_large) { + mi_memid_t memid = _mi_memid_create(MI_MEM_OS); + memid.mem.os.base = base; + memid.mem.os.size = size; + memid.initially_committed = committed; + memid.initially_zero = is_zero; + memid.is_pinned = is_large; + return memid; +} + +mi_memid_t _mi_memid_create_static(void* p, size_t size) { + mi_memid_t memid = _mi_memid_create(MI_MEM_STATIC); + memid.mem.malloc.base = p; + memid.mem.malloc.size = size; + memid.initially_committed = true; + memid.is_pinned = true; + return memid; +} + +mi_memid_t _mi_memid_create_malloc(void* p, size_t size, bool iszero) { + mi_memid_t memid = _mi_memid_create(MI_MEM_MALLOC); + memid.mem.malloc.base = p; + memid.mem.malloc.size = size; + memid.initially_committed = true; + memid.initially_zero = iszero; + memid.is_pinned = true; + return memid; +} + +size_t _mi_memid_size(mi_memid_t memid) { + if (mi_memid_is_os(memid)) { + return memid.mem.os.size; + } + else if (memid.memkind == MI_MEM_ARENA) { + return _mi_size_of_slices(memid.mem.arena.slice_count); + } + else if (memid.memkind == MI_MEM_MALLOC) { + return memid.mem.malloc.size; + } + else { + mi_assert_internal(mi_memid_needs_no_free(memid)); + return 0; + } +} + + +/* ----------------------------------------------------------- + Arena id's +----------------------------------------------------------- */ + +mi_arena_id_t _mi_arena_id_none(void) { + return NULL; +} + +mi_arena_t* _mi_arena_from_id(mi_arena_id_t id) { + mi_arena_t* const arena = (mi_arena_t*)id; + mi_assert_internal(arena==NULL || arena->parent==NULL); // id's should never point to sub-arena's + return arena; +} + +mi_arena_id_t mi_arena_id_from_arena(mi_arena_t* arena) { + mi_assert_internal(arena==NULL || arena->parent==NULL); + return (arena==NULL ? _mi_arena_id_none() : (mi_arena_id_t)arena); +} + + +bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_t* request_arena) { + if (memid.memkind == MI_MEM_ARENA) { + return mi_arena_is_suitable(memid.mem.arena.arena, request_arena); + } + else { + return mi_arena_is_suitable(NULL, request_arena); + } +} + +size_t mi_arena_min_alignment(void) { + return MI_ARENA_ALIGNMENT; +} + +size_t mi_arena_min_size(void) { + return MI_ARENA_MIN_SIZE; +} + +// fixed limit for the maximum object size in an arena +static size_t mi_arena_max_fixed_object_size(void) { + #if MI_PAGE_META_IS_ALIGNED + return (MI_PAGE_META_ALIGNMENT - mi_align_up(MI_PAGE_META_ALIGNED_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE)); + #else + return (MI_ARENA_MAX_SIZE - MI_ARENA_CHUNK_SIZE); // minus an initial chunk to accommodate meta info + #endif +} + +// Maximum object size allowed to be allocated in an arena +size_t mi_arena_max_object_size(void) { + size_t max_size = mi_option_get_size(mi_option_arena_max_object_size); + max_size = mi_align_up(max_size, MI_ARENA_SLICE_SIZE); + if (max_size <= MI_ARENA_MIN_OBJ_SIZE) { + return MI_ARENA_MIN_OBJ_SIZE; + } + else if (max_size >= mi_arena_max_fixed_object_size()) { + return mi_arena_max_fixed_object_size(); + } + else { + return max_size; + } +} + +// Size of an arena +static size_t mi_arena_size(mi_arena_t* arena) { + return _mi_size_of_slices(arena->slice_count); +} + + +// Arena area +void* mi_arena_area(mi_arena_id_t arena_id, size_t* size) { + if (size != NULL) *size = 0; + mi_arena_t* arena = _mi_arena_from_id(arena_id); + if (arena == NULL) return NULL; + if (size != NULL) { + mi_assert_internal(_mi_size_of_slices(arena->slice_count) <= arena->total_size); + *size = arena->total_size; + } + return mi_arena_start(arena); +} + + + +/* ----------------------------------------------------------- + Arena iteration +----------------------------------------------------------- */ + +static bool mi_arena_is_suitable_ex(mi_arena_t* arena, mi_arena_t* req_arena, bool match_numa, int numa_node, bool allow_pinned) { + if (!allow_pinned && arena->memid.is_pinned) return false; + if (!mi_arena_is_suitable(arena, req_arena)) return false; + if (req_arena == NULL) { // if not specific, check numa affinity + const bool numa_suitable = (numa_node < 0 || arena->numa_node < 0 || arena->numa_node == numa_node); + if (match_numa) { if (!numa_suitable) return false; } + else { if (numa_suitable) return false; } + } + return true; +} + +// determine the start of search; important to keep heaps and threads +// into their own memory regions to reduce contention. +static inline size_t mi_arena_start_idx(mi_heap_t* heap, size_t tseq, size_t arena_cycle) +{ + const size_t hseq = heap->heap_seq; + const size_t hcount = mi_atomic_load_relaxed(&heap->subproc->heap_count); + if (arena_cycle <= 1) return 0; + if (hseq==0 || hcount<=1 || arena_cycle > 0x8FF) return (tseq % arena_cycle); // common for single heap programs + + // spread heaps evenly among arena's, and then evenly for threads in their fraction + size_t start; + mi_assert_internal(arena_cycle <= 0x8FF); // prevent overflow on 32-bit + const size_t frac = (arena_cycle * 256) / hcount; // fraction in the arena_cycle; at most: arena_cycle * 0x100 + if (frac==0) { + // many heaps (> 256 per arena) + start = (hseq % arena_cycle); + } + else { + const size_t hspot = (hseq % hcount); + start = (frac * hspot) / 256; // (arena_cycle * (hseq % hcount)) / hcount + if (frac >= 512) { // at least 2 arena's per heap? + start = start + (tseq % (frac/256)); + } + } + mi_assert_internal(start < arena_cycle); + return start; +} + +mi_decl_forceinline bool mi_forall_arenas(mi_heap_t* heap, mi_arena_t* req_arena, size_t tseq, mi_forall_arena_fun_t* visit, const void* arg, void** result) +{ + if (result != NULL) *result = NULL; + const size_t arena_count = mi_arenas_get_count(heap->subproc); + const size_t arena_cycle = (arena_count == 0 ? 0 : arena_count - 1); /* first search the arenas below the last one */ + /* always start searching in the arena's below the max */ + const size_t start = (req_arena==NULL ? mi_arena_start_idx(heap,tseq,arena_cycle) : req_arena->arena_idx); + mi_assert_internal(start <= arena_count); + for (size_t i = 0; i < arena_count; i++) { + size_t idx; + if (i < arena_cycle) { + idx = i + start; + if (idx >= arena_cycle) { idx -= arena_cycle; } /* adjust so we rotate through the cycle */ + } + else { + idx = i; /* remaining arena's after the cycle */ + } + mi_arena_t* arena = mi_arena_from_index(heap->subproc,idx); + if (req_arena != NULL && arena != req_arena && + (arena == NULL || arena->parent != req_arena)) continue; /* only the requested arena or its children */ + + // call the visitor + if (arena != NULL) { + if (!visit(arena, arg, result)) return false; + } + } + return true; +} + +typedef struct mi_arena_suitable_visit_info_s { + mi_arena_t* req_arena; + bool match_numa; + int numa_node; + bool allow_large; + mi_forall_arena_fun_t* visit; + const void* arg; +} mi_arena_suitable_visit_info_t; + +static inline bool mi_arena_visit_suitable(mi_arena_t* arena, const void* arg, void** result) +{ + const mi_arena_suitable_visit_info_t* info = (const mi_arena_suitable_visit_info_t*)arg; + if (!mi_arena_is_suitable_ex(arena, info->req_arena, info->match_numa, info->numa_node, info->allow_large)) { + return true; // skip and continue + } + else { + return info->visit(arena, info->arg, result); // visit + } +} + +mi_decl_forceinline bool mi_forall_suitable_arenas(mi_heap_t* heap, mi_arena_t* req_arena, size_t tseq, bool match_numa, int numa_node, + bool allow_large, mi_forall_arena_fun_t* visit, const void* arg, void** result) { + const mi_arena_suitable_visit_info_t info = { req_arena, match_numa, numa_node, allow_large, visit, arg }; + return mi_forall_arenas(heap, req_arena, tseq, &mi_arena_visit_suitable, &info, result); +} + +#if defined(__cplusplus) || defined(_MSC_VER) +// since these are mi_decl_forceinline, ensure they are linked in the final binary by adding a reference. +void* _mi_arena_externs[] = { + (void*)&mi_forall_arenas, + (void*)&mi_forall_suitable_arenas +}; +#endif + + +/* ----------------------------------------------------------- + Reserve a fresh arena +----------------------------------------------------------- */ + +static int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id); + +// try to reserve a fresh arena space +bool _mi_arena_reserve(mi_subproc_t* subproc, size_t req_size, bool allow_large, mi_arena_id_t* arena_id) +{ + const size_t arena_count = mi_arenas_get_count(subproc); + if (arena_count > (MI_MAX_ARENAS - 4)) return false; + + // calc reserve + size_t arena_reserve = mi_option_get_size(mi_option_arena_reserve); + if (arena_reserve == 0) return false; + + if (!_mi_os_has_virtual_reserve()) { + arena_reserve = arena_reserve/4; // be conservative if virtual reserve is not supported (for WASM for example) + } + arena_reserve = mi_align_up(arena_reserve, MI_ARENA_SLICE_SIZE); + + if (arena_count >= 1 && arena_count <= 128) { + // scale up the arena sizes exponentially every 8 entries + const size_t multiplier = (size_t)1 << mi_clamp(arena_count/8, 0, 16); + size_t reserve = 0; + if (!mi_mul_overflow(multiplier, arena_reserve, &reserve)) { + arena_reserve = reserve; + } + } + + // try to accommodate the requested size for huge allocations + req_size = mi_align_up(req_size + MI_ARENA_MAX_CHUNK_OBJ_SIZE, MI_ARENA_MAX_CHUNK_OBJ_SIZE); // over-reserve for meta-info + if (arena_reserve < req_size) { + arena_reserve = req_size; + } + + // check arena bounds + const size_t min_reserve = MI_ARENA_MIN_SIZE; + const size_t max_reserve = MI_ARENA_MAX_SIZE; // 16 GiB + if (arena_reserve < min_reserve) { + arena_reserve = min_reserve; + } + else if (arena_reserve > max_reserve) { + arena_reserve = max_reserve; + } + + // should be able to at least handle the current allocation size + if (arena_reserve < req_size) return false; + + // commit eagerly? + bool arena_commit = false; + const bool overcommit = _mi_os_has_overcommit(); + if (mi_option_get(mi_option_arena_eager_commit) == 2) { arena_commit = overcommit || mi_option_is_enabled(mi_option_allow_large_os_pages); } + else if (mi_option_get(mi_option_arena_eager_commit) == 1) { arena_commit = true; } + + // on an OS with overcommit (Linux) we don't count the commit yet as it is on-demand. Once a slice + // is actually allocated for the first time it will be counted. + const bool adjust = (overcommit && arena_commit); + if (adjust) { mi_subproc_stat_adjust_decrease( subproc, committed, arena_reserve); } + // and try to reserve the arena + int err = mi_reserve_os_memory_ex2(subproc, arena_reserve, arena_commit, allow_large, false /* exclusive? */, arena_id); + if (err != 0) { + if (adjust) { mi_subproc_stat_adjust_increase( subproc, committed, arena_reserve); } // roll back + // failed to allocate: try a smaller size arena as fallback? + const size_t small_arena_reserve = 4 * MI_ARENA_MIN_SIZE; // 128 MiB (or 32 MiB on 32-bit) + if (arena_reserve > small_arena_reserve && small_arena_reserve > req_size) { + // try again + if (adjust) { mi_subproc_stat_adjust_decrease(subproc, committed, small_arena_reserve); } + err = mi_reserve_os_memory_ex2(subproc, small_arena_reserve, arena_commit, allow_large, false /* exclusive? */, arena_id); + if (err != 0 && adjust) { mi_subproc_stat_adjust_increase( subproc, committed, small_arena_reserve); } // roll back + } + } + return (err==0); +} + + +// Is a pointer contained in the given arena area? +static bool mi_arena_strictly_contains(mi_arena_t* arena, const void* p) { + return (arena != NULL && + mi_arena_start(arena) <= (const uint8_t*)p && + mi_arena_start(arena) + _mi_size_of_slices(arena->slice_count) >(const uint8_t*)p); +} + +// Is a pointer inside any of our arenas? +static bool mi_arenas_contain_ex(const void* p, mi_arena_t* parent) { + mi_subproc_t* subproc = _mi_subproc(); + const size_t max_arena = mi_arenas_get_count(subproc); + for (size_t i = 0; i < max_arena; i++) { + mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]); + if (arena != NULL) { + if (parent==NULL || arena==parent || arena->parent==parent) { + if (mi_arena_strictly_contains(arena, p)) { + return true; + } + } + } + } + return false; +} + +// // Is a pointer inside any of our arenas? +// bool _mi_arenas_contain(const void* p) { +// return mi_arenas_contain_ex(p, NULL); +// } + +// Is a pointer contained in the given arena area? +bool mi_arena_contains(mi_arena_id_t arena_id, const void* p) { + mi_arena_t* arena = _mi_arena_from_id(arena_id); + if (arena==NULL) return false; + else if (mi_arena_strictly_contains(arena, p)) return true; + else return mi_arenas_contain_ex(p, arena); // maybe a subarena? +} + + +/* ----------------------------------------------------------- + Remove an arena. +----------------------------------------------------------- */ + +// destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit` +// for dynamic libraries that are unloaded and need to release all their allocated memory. +static void mi_arenas_unsafe_destroy(mi_subproc_t* subproc) { + mi_assert_internal(subproc != NULL); + const size_t arena_count = mi_arenas_get_count(subproc); + for (size_t i = 0; i < arena_count; i++) { + mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]); + if (arena != NULL) { + // mi_lock_done(&arena->abandoned_visit_lock); + mi_atomic_store_ptr_release(mi_arena_t, &subproc->arenas[i], NULL); + if (mi_memkind_is_os(arena->memid.memkind)) { + _mi_os_free_ex(subproc, mi_arena_start(arena), mi_arena_size(arena), true, arena->memid); + } + } + } + // try to lower the max arena. + size_t expected = arena_count; + mi_atomic_cas_strong_acq_rel(&subproc->arena_count, &expected, (size_t)0); +} + + +// destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit` +// for dynamic libraries that are unloaded and need to release all their allocated memory. +void _mi_arenas_unsafe_destroy_all(mi_subproc_t* subproc) { + mi_arenas_unsafe_destroy(subproc); + // mi_arenas_try_purge(true /* force purge */, true /* visit all*/, subproc, 0 /* thread seq */); // purge non-owned arenas +} + + +/* ----------------------------------------------------------- + Add an arena. +----------------------------------------------------------- */ + +static bool mi_arenas_add(mi_subproc_t* subproc, mi_arena_t* arena, mi_arena_id_t* arena_id) +{ + mi_assert_internal(arena != NULL); + mi_assert_internal(arena->slice_count > 0); + if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); } + + // try to find a NULL entry + mi_arena_t* expected; + size_t count = mi_arenas_get_count(subproc); + for( size_t i = 0; i < count; i++) { + if (mi_arena_from_index(subproc,i) == NULL) { + arena->arena_idx = i; + expected = NULL; + if (mi_atomic_cas_ptr_strong_release(mi_arena_t, &subproc->arenas[i], &expected, arena)) { + // success + if (arena_id != NULL) { *arena_id = mi_arena_id_from_arena(arena); } + return true; + } + } + } + + // otherwise, try to allocate a fresh slot + while(countarena_count, &count, count+1)) { + arena->arena_idx = count; + expected = NULL; + if (mi_atomic_cas_ptr_strong_release(mi_arena_t, &subproc->arenas[count], &expected, arena)) { + mi_subproc_stat_counter_increase(arena->subproc, arena_count, 1); + if (arena_id != NULL) { *arena_id = mi_arena_id_from_arena(arena); } + return true; + } + } + } + + // failed + arena->arena_idx = 0; + arena->subproc = NULL; + return false; +} + + +static mi_arena_t* mi_arena_info(void* area) { + return (mi_arena_t*)((uint8_t*)area + _mi_size_of_slices(mi_arena_page_meta_aligned_slice_count())); +} + +static size_t mi_arena_info_slices_needed(size_t slice_count, size_t* bitmap_base) { + if (slice_count == 0) slice_count = MI_BCHUNK_BITS; + mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0); + const size_t base_size = _mi_size_of_slices(mi_arena_page_meta_aligned_slice_count()) + mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE); + const size_t bitmaps_count = 4 + MI_ARENA_BIN_COUNT; // commit, dirty, purge, pages, and abandoned + const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL) + mi_bbitmap_size(slice_count, NULL); // + free + #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_IS_ALIGNED + const size_t pages_size = slice_count * sizeof(mi_page_t); + #else + const size_t pages_size = 0; + #endif + const size_t size = base_size + bitmaps_size + pages_size; + + const size_t os_page_size = _mi_os_page_size(); + const size_t info_size = mi_align_up(size, os_page_size) + _mi_os_secure_guard_page_size(); + const size_t info_slices = _mi_slice_count_of_size(info_size); + + if (bitmap_base != NULL) *bitmap_base = base_size; + return info_slices; +} + +static mi_bbitmap_t* mi_arena_bbitmap_init(mi_subproc_t* subproc, size_t slice_count, uint8_t** base) { + mi_bbitmap_t* bbitmap = (mi_bbitmap_t*)(*base); + *base = (*base) + mi_bbitmap_init(subproc, bbitmap, slice_count, true /* already zero */); + return bbitmap; +} + + +static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, + size_t slice_count, mi_arena_t* parent, size_t total_size, + int numa_node, bool exclusive, + mi_memid_t memid, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) +{ + mi_assert_internal(mi_is_aligned(start,MI_ARENA_ALIGNMENT)); + mi_assert_internal(_mi_size_of_slices(slice_count)>=MI_ARENA_MIN_SIZE); + + if (slice_count > MI_BITMAP_MAX_BIT_COUNT) { // 16 GiB for now + // note: this should never happen if called from `mi_manage_os_memory` (as that allocates sub-arenas when needed) + _mi_warning_message("cannot use OS memory since it is too large (size %zu MiB, maximum is %zu MiB)", _mi_size_of_slices(slice_count)/MI_MiB, _mi_size_of_slices(MI_BITMAP_MAX_BIT_COUNT)/MI_MiB); + return NULL; + } + + size_t bitmap_base; + const size_t info_slices = mi_arena_info_slices_needed(slice_count, &bitmap_base); + if (slice_count < info_slices+1) { + _mi_warning_message("cannot use OS memory since it is not large enough (size %zu KiB, minimum required is %zu KiB)", _mi_size_of_slices(slice_count)/MI_KiB, _mi_size_of_slices(info_slices+1)/MI_KiB); + return NULL; + } + // else if (info_slices >= MI_ARENA_MAX_CHUNK_OBJ_SLICES) { + // _mi_warning_message("cannot use OS memory since it is too large with respect to the maximum object size (size %zu MiB, meta-info slices %zu, maximum object slices are %zu)", _mi_size_of_slices(slice_count)/MI_MiB, info_slices, MI_ARENA_MAX_CHUNK_OBJ_SLICES); + // return NULL; + // } + + // commit & zero if needed + if (!memid.initially_committed) { + size_t commit_size = _mi_size_of_slices(info_slices); + // leave a guard OS page decommitted at the end? + if (!memid.is_pinned) { commit_size -= _mi_os_secure_guard_page_size(); } + bool ok = false; + if (commit_fun != NULL) { + ok = (*commit_fun)(true /* commit */, start, commit_size, NULL, commit_fun_arg); + } + else { + ok = _mi_os_commit(subproc, start, commit_size, NULL); + } + if (!ok) { + _mi_warning_message("unable to commit meta-data for OS memory"); + return NULL; + } + } + else if (!memid.is_pinned) { + // if MI_SECURE, set a guard page at the end of the arena info + // todo: this does not respect the commit_fun as the memid is of external memory + _mi_os_secure_guard_page_set_before(subproc, (uint8_t*)start + _mi_size_of_slices(info_slices), memid); + } + if (!memid.initially_zero) { + mi_memzero(start, _mi_size_of_slices(info_slices) - _mi_os_secure_guard_page_size()); + } + + // init + mi_arena_t* arena = mi_arena_info(start); + arena->start = start; + arena->subproc = subproc; + arena->memid = memid; + arena->is_exclusive = exclusive; + arena->slice_count = slice_count; + arena->info_slices = info_slices; + if (numa_node<0 && mi_option_is_enabled(mi_option_arena_is_numa_local)) { + arena->numa_node = _mi_os_numa_node(); + } + else { + arena->numa_node = numa_node; + } + arena->purge_expire = 0; + arena->commit_fun = commit_fun; + arena->commit_fun_arg = commit_fun_arg; + arena->parent = parent; + arena->total_size = total_size; + + // init bitmaps + uint8_t* base = mi_arena_start(arena) + bitmap_base; + arena->slices_free = mi_arena_bbitmap_init(subproc, slice_count, &base); + arena->slices_committed = mi_arena_bitmap_init(slice_count, &base); + arena->slices_dirty = mi_arena_bitmap_init(slice_count, &base); + arena->slices_purge = mi_arena_bitmap_init(slice_count, &base); + arena->pages_main.pages = mi_arena_bitmap_init(slice_count, &base); + for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) { + arena->pages_main.pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base); + } + #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_IS_ALIGNED + arena->pages_meta = (mi_page_t*)base; + base += (slice_count * sizeof(mi_page_t)); + #else + arena->pages_meta = NULL; + #endif + mi_assert_internal(_mi_size_of_slices(info_slices) >= (size_t)(base - mi_arena_start(arena))); + + // reserve our meta info (and reserve slices outside the memory area) + #if MI_PAGE_META_IS_ALIGNED + for(size_t i = 0; i < arena->slice_count; i += MI_PAGE_META_ALIGNED_COUNT) { + // set all free slices (and skip the slices reserved for the page meta info) + const size_t meta_slices = (i==0 ? info_slices : mi_arena_page_meta_aligned_slice_count()); + const size_t start_idx = (i + meta_slices); + size_t count = MI_PAGE_META_ALIGNED_COUNT - meta_slices; + if (start_idx < arena->slice_count) { + if (count + start_idx > arena->slice_count) { + count = arena->slice_count - start_idx; + mi_assert_internal(count > 0); + } + mi_bbitmap_unsafe_setN(arena->slices_free, start_idx, count); + } + } + #else + mi_bbitmap_unsafe_setN(arena->slices_free, info_slices /* start */, arena->slice_count - info_slices); + #endif + if (memid.initially_committed) { + mi_bitmap_unsafe_setN(arena->slices_committed, 0, arena->slice_count); + } + if (!memid.initially_zero) { + mi_bitmap_unsafe_setN(arena->slices_dirty, 0, arena->slice_count); + } + + if (!mi_arenas_add(subproc, arena, arena_id)) { return NULL; } + return arena; +} + + +/* ----------------------------------------------------------- + Reserve and manage OS memory (as an arena) +----------------------------------------------------------- */ + +static bool mi_manage_os_memory_ex2(mi_subproc_t* subproc, void* start, size_t size, int numa_node, bool exclusive, + mi_memid_t memid, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) mi_attr_noexcept +{ + // checks + mi_assert(start!=NULL); + if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); } + if (start==NULL) return false; + if (!mi_is_aligned(start, MI_ARENA_ALIGNMENT)) { + // we can align the start since the memid tracks the real base of the memory. + void* const aligned_start = mi_align_up_ptr(start, MI_ARENA_ALIGNMENT); + const size_t diff = (uint8_t*)aligned_start - (uint8_t*)start; + if (diff >= size || (size - diff) < MI_ARENA_ALIGNMENT) { + _mi_warning_message("after alignment, the size of the arena becomes too small (memory at %p with size %zu)\n", start, size); + return false; + } + start = aligned_start; + size = size - diff; + } + + // allocate enough arena's to span the full memory area + // the first arena is the owner, the rest are "sub-arena" (with `parent` pointing to the first one) + size_t total_slice_count = mi_align_down(size / MI_ARENA_SLICE_SIZE, MI_BCHUNK_BITS); + size_t total_size = _mi_size_of_slices(total_slice_count); + if (total_size < MI_ARENA_MIN_SIZE) { + _mi_warning_message("cannot use OS memory since it is not large enough (size %zu KiB, minimum required is %zu KiB)", size/MI_KiB, MI_ARENA_MIN_SIZE/MI_KiB); + return false; + } + + mi_arena_t* parent = NULL; + do { + // counting down on the total_slice_count + size_t slice_count = total_slice_count; + if (slice_count > MI_BITMAP_MAX_BIT_COUNT) { // 16 GiB for now (with 64KiB slices) + slice_count = MI_BITMAP_MAX_BIT_COUNT; + } + + // initialize + mi_arena_t* arena = mi_arena_initialize( subproc, start, slice_count, parent, + (parent==NULL ? total_size : 0), numa_node, exclusive, + memid, commit_fun, commit_fun_arg, + (parent==NULL ? arena_id : NULL)); + if (arena==NULL) { + // failed to initialize due to failing commit or too many arena's + if (parent==NULL) { + return false; + } + else { + // partial success, but failed to use the full area.. + // todo: roll-back in this case? that requires a lock on the arena's array though + mi_assert(_mi_size_of_slices(total_slice_count) <= parent->total_size); + parent->total_size -= _mi_size_of_slices(total_slice_count); + return true; + } + } + + // success + if (parent==NULL) { + parent = arena; + memid.memkind = MI_MEM_NONE; + } + mi_assert(slice_count <= total_slice_count); + total_slice_count -= slice_count; + start = (uint8_t*)start + _mi_size_of_slices(slice_count); + } + while (total_slice_count > 0); + + return true; +} + +bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept { + mi_memid_t memid = _mi_memid_create(MI_MEM_EXTERNAL); + memid.mem.os.base = start; + memid.mem.os.size = size; + memid.initially_committed = is_committed; + memid.initially_zero = is_zero; + memid.is_pinned = is_pinned; + return mi_manage_os_memory_ex2(_mi_subproc(), start, size, numa_node, exclusive, memid, NULL, NULL, arena_id); +} + +bool mi_manage_memory(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) mi_attr_noexcept +{ + mi_memid_t memid = _mi_memid_create(MI_MEM_EXTERNAL); + memid.mem.os.base = start; + memid.mem.os.size = size; + memid.initially_committed = is_committed; + memid.initially_zero = is_zero; + memid.is_pinned = is_pinned; + return mi_manage_os_memory_ex2(_mi_subproc(), start, size, numa_node, exclusive, memid, commit_fun, commit_fun_arg, arena_id); +} + + +// Reserve a range of regular OS memory +static int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) { + if (arena_id != NULL) *arena_id = _mi_arena_id_none(); + if (size <= MI_MAX_ALLOC_SIZE) { + size = mi_align_up(size, MI_ARENA_SLICE_SIZE); // at least one slice + } + if (size > MI_MAX_ALLOC_SIZE) { + _mi_error_message(EOVERFLOW, "memory reservation request is too large (size %zu)\n", size); + return ENOMEM; + } + mi_memid_t memid; + void* start = _mi_os_alloc_aligned(subproc, size, MI_ARENA_ALIGNMENT, commit, allow_large, &memid); + if (start == NULL) return ENOMEM; + if (!mi_manage_os_memory_ex2(subproc, start, size, -1 /* numa node */, exclusive, memid, NULL, NULL, arena_id)) { + _mi_os_free_ex(subproc, start, size, commit, memid); + _mi_verbose_message("failed to reserve %zu KiB memory\n", mi_divide_up(size, 1024)); + return ENOMEM; + } + _mi_verbose_message("reserved %zu KiB memory%s\n", mi_divide_up(size, 1024), memid.is_pinned ? " (in large os pages)" : ""); + // mi_debug_show_arenas(true, true, false); + + return 0; +} + +// Reserve a range of regular OS memory +int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept { + return mi_reserve_os_memory_ex2(_mi_subproc(), size, commit, allow_large, exclusive, arena_id); +} + +// Manage a range of regular OS memory +bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept { + return mi_manage_os_memory_ex(start, size, is_committed, is_large, is_zero, numa_node, false /* exclusive? */, NULL); +} + +// Reserve a range of regular OS memory +int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept { + return mi_reserve_os_memory_ex(size, commit, allow_large, false, NULL); +} + + +/* ----------------------------------------------------------- + Debugging +----------------------------------------------------------- */ + +// Return idx of the slice past the last used slice +static size_t mi_arena_used_slices(mi_arena_t* arena) { + size_t idx; + if (mi_bbitmap_bsr_inv(arena->slices_free, &idx)) { + return (idx + 1); + } + else { + return mi_arena_info_slices(arena); + } +} + +static size_t mi_debug_show_bfield(mi_bfield_t field, char* buf, size_t* k) { + size_t bit_set_count = 0; + for (int bit = 0; bit < MI_BFIELD_BITS; bit++) { + bool is_set = ((((mi_bfield_t)1 << bit) & field) != 0); + if (is_set) bit_set_count++; + buf[*k] = (is_set ? 'x' : '.'); + *k = *k + 1; + } + return bit_set_count; +} + +typedef enum mi_ansi_color_e { + MI_BLACK = 30, + MI_MAROON, + MI_DARKGREEN, + MI_ORANGE, + MI_NAVY, + MI_PURPLE, + MI_TEAL, + MI_GRAY, + MI_DARKGRAY = 90, + MI_RED, + MI_GREEN, + MI_YELLOW, + MI_BLUE, + MI_MAGENTA, + MI_CYAN, + MI_WHITE +} mi_ansi_color_t; + +static void mi_debug_color(char* buf, size_t* k, mi_ansi_color_t color) { + *k += _mi_snprintf(buf + *k, 32, "\x1B[%dm", (int)color); +} + +static int mi_page_commit_usage(mi_page_t* page) { + const size_t committed_size = mi_page_committed(page); + const size_t used_size = mi_page_used(page) * mi_page_block_size(page); + return (int)(used_size * 100 / committed_size); +} + +static size_t mi_debug_show_page_bfield(char* buf, size_t* k, mi_arena_t* arena, size_t slice_index, long* pbit_of_page, mi_ansi_color_t* pcolor_of_page ) { + size_t bit_set_count = 0; + long bit_of_page = *pbit_of_page; + mi_ansi_color_t color = *pcolor_of_page; + mi_ansi_color_t prev_color = MI_GRAY; + for (int bit = 0; bit < MI_BFIELD_BITS; bit++, bit_of_page--) { + // bool is_set = ((((mi_bfield_t)1 << bit) & field) != 0); + void* start = mi_arena_slice_start(arena, slice_index + bit); + mi_page_t* page = _mi_safe_ptr_page(start); + char c = ' '; + if (page!=NULL && start==mi_page_slice_start(page)) { + mi_assert_internal(bit_of_page <= 0); + bit_set_count++; + c = 'p'; + color = MI_GRAY; + if (_mi_meta_is_meta_page(arena->subproc,page)) { c = 'm'; } + else if (mi_page_is_singleton(page)) { c = 's'; } + else if (mi_page_is_full(page)) { c = 'f'; } + if (!mi_page_is_abandoned(page)) { c = _mi_toupper(c); } + int commit_usage = mi_page_commit_usage(page); + if (commit_usage < 25) { color = MI_MAROON; } + else if (commit_usage < 50) { color = MI_ORANGE; } + else if (commit_usage < 75) { color = MI_TEAL; } + else color = MI_DARKGREEN; + bit_of_page = (long)page->memid.mem.arena.slice_count; + } + else { + c = '?'; + if (bit_of_page > 0) { c = '-'; } + // else if (_mi_meta_is_meta_page(arena->subproc,start)) { c = 'm'; color = MI_GRAY; } + else if (slice_index + bit < arena->info_slices) { c = 'i'; color = MI_GRAY; } + #if MI_PAGE_META_IS_ALIGNED + else if ((slice_index % MI_PAGE_META_ALIGNED_COUNT) == 0 && (size_t)bit <= mi_arena_page_meta_aligned_slice_count()) { + { c = 'i'; color = MI_GRAY; } + } + #endif + // else if (mi_bitmap_is_setN(arena->pages_purge, slice_index + bit, NULL)) { c = '*'; } + else if (mi_bbitmap_is_setN(arena->slices_free, slice_index+bit,1)) { + if (mi_bitmap_is_set(arena->slices_purge, slice_index + bit)) { c = '~'; color = MI_ORANGE; } + else if (mi_bitmap_is_set(arena->slices_committed, slice_index + bit)) { c = '_'; color = MI_GRAY; } + else { c = '.'; color = MI_GRAY; } + } + if (bit==MI_BFIELD_BITS-1 && bit_of_page > 1) { c = '>'; } + } + if (color != prev_color) { + mi_debug_color(buf, k, color); + prev_color = color; + } + buf[*k] = c; *k += 1; + } + mi_debug_color(buf, k, MI_GRAY); + *pbit_of_page = bit_of_page; + *pcolor_of_page = color; + return bit_set_count; +} + +static size_t mi_debug_show_chunks(const char* header1, const char* header2, const char* header3, + size_t slice_count, size_t chunk_count, + mi_bchunk_t* chunks, mi_bchunkmap_t* chunk_bins, bool invert, mi_arena_t* arena, bool narrow) +{ + _mi_raw_message("\x1B[37m%s%s%s (use/commit: \x1B[31m0 - 25%%\x1B[33m - 50%%\x1B[36m - 75%%\x1B[32m - 100%%\x1B[0m)\n", header1, header2, header3); + const size_t fields_per_line = (narrow ? 2 : 4); + const size_t used_slice_count = mi_arena_used_slices(arena); + size_t bit_count = 0; + size_t bit_set_count = 0; + long bit_of_page = 0; + mi_ansi_color_t color_of_page = MI_GRAY; + for (size_t i = 0; i < chunk_count && bit_count < slice_count; i++) { + char buf[5*MI_BCHUNK_BITS + 64]; mi_memzero(buf, sizeof(buf)); + if (bit_count > used_slice_count && i+2 < chunk_count) { + const size_t diff = chunk_count - 1 - i; + bit_count += diff*MI_BCHUNK_BITS; + _mi_raw_message(" |\n"); + i = chunk_count-1; + } + + size_t k = 0; + + if (i<10) { buf[k++] = ('0' + (char)i); buf[k++] = ' '; buf[k++] = ' '; } + else if (i<100) { buf[k++] = ('0' + (char)(i/10)); buf[k++] = ('0' + (char)(i%10)); buf[k++] = ' '; } + else if (i<1000) { buf[k++] = ('0' + (char)(i/100)); buf[k++] = ('0' + (char)((i%100)/10)); buf[k++] = ('0' + (char)(i%10)); } + + char chunk_kind = ' '; + if (chunk_bins != NULL) { + switch (mi_bbitmap_debug_get_bin(chunk_bins,i)) { + case MI_CBIN_SMALL: chunk_kind = 'S'; break; + case MI_CBIN_MEDIUM: chunk_kind = 'M'; break; + case MI_CBIN_LARGE: chunk_kind = 'L'; break; + case MI_CBIN_HUGE: chunk_kind = 'H'; break; + case MI_CBIN_OTHER: chunk_kind = 'X'; break; + default: chunk_kind = ' '; break; // suppress warning + // case MI_CBIN_NONE: chunk_kind = 'N'; break; + } + } + buf[k++] = chunk_kind; + buf[k++] = ' '; + + for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) { + if (j > 0 && (j % fields_per_line) == 0) { + // buf[k++] = '\n'; mi_memset(buf+k,' ',7); k += 7; + _mi_raw_message(" %s\n\x1B[37m", buf); + mi_memzero(buf, sizeof(buf)); + mi_memset(buf, ' ', 5); k = 5; + } + if (bit_count < slice_count) { + mi_bfield_t bfield = 0; + if (chunks!=NULL) { + bfield = chunks[i].bfields[j]; + } + if (invert) bfield = ~bfield; + size_t xcount = (chunks==NULL ? mi_debug_show_page_bfield(buf, &k, arena, bit_count, &bit_of_page, &color_of_page) + : mi_debug_show_bfield(bfield, buf, &k)); + if (invert) xcount = MI_BFIELD_BITS - xcount; + bit_set_count += xcount; + buf[k++] = ' '; + } + else { + mi_memset(buf + k, 'o', MI_BFIELD_BITS); + k += MI_BFIELD_BITS; + } + bit_count += MI_BFIELD_BITS; + } + _mi_raw_message(" %s\n\x1B[37m", buf); + } + _mi_raw_message("\x1B[0m total pages: %zu\n", bit_set_count); + return bit_set_count; +} + +//static size_t mi_debug_show_bitmap_binned(const char* header1, const char* header2, const char* header3, size_t slice_count, +// mi_bitmap_t* bitmap, mi_bchunkmap_t* chunk_bins, bool invert, mi_arena_t* arena, bool narrow) { +// return mi_debug_show_chunks(header1, header2, header3, slice_count, mi_bitmap_chunk_count(bitmap), &bitmap->chunks[0], chunk_bins, invert, arena, narrow); +//} + +static void mi_debug_show_arenas_ex(mi_heap_t* heap, bool show_pages, bool narrow) mi_attr_noexcept { + mi_subproc_t* subproc = heap->subproc; + size_t max_arenas = mi_arenas_get_count(subproc); + //size_t free_total = 0; + //size_t slice_total = 0; + //size_t abandoned_total = 0; + size_t page_total = 0; + for (size_t i = 0; i < max_arenas; i++) { + mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]); + if (arena == NULL) continue; + mi_assert(arena->subproc == subproc); + // slice_total += arena->slice_count; + _mi_raw_message("%sarena %zu at %p: %zu slices (%zu MiB)%s%s, subproc: %zu, numa: %i\n", + (arena->parent==NULL ? "" : "(sub)"), i, arena, arena->slice_count, (size_t)(_mi_size_of_slices(arena->slice_count)/MI_MiB), + (arena->memid.is_pinned ? ", pinned" : ""), (arena->is_exclusive ? ", exclusive" : ""), + arena->subproc->subproc_seq, arena->numa_node); + //if (show_inuse) { + // free_total += mi_debug_show_bbitmap("in-use slices", arena->slice_count, arena->slices_free, true, NULL); + //} + //if (show_committed) { + // mi_debug_show_bitmap("committed slices", arena->slice_count, arena->slices_committed, false, NULL); + //} + // todo: abandoned slices + //if (show_purge) { + // purge_total += mi_debug_show_bitmap("purgeable slices", arena->slice_count, arena->slices_purge, false, NULL); + //} + if (show_pages) { + // mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena); + // if (arena_pages != NULL) + { + const char* header1 = "chunks (p:page, f:full, s:single, m:meta-data, i:arena-info, P,F,S,M:not abandoned, ~:free-purgable, _:free-committed, .:free-reserved)"; + const char* header2 = (narrow ? "\n " : " "); + const char* header3 = "(chunk bin: S:small, M : medium, L : large, X : other)"; + page_total += mi_debug_show_chunks(header1, header2, header3, arena->slice_count, + mi_bbitmap_chunk_count(arena->slices_free), NULL, + arena->slices_free->chunkmap_bins, false, arena, narrow); + } + } + } + // if (show_inuse) _mi_raw_message("total inuse slices : %zu\n", slice_total - free_total); + // if (show_abandoned) _mi_raw_message("total abandoned slices: %zu\n", abandoned_total); + if (show_pages) _mi_raw_message("total pages in arenas: %zu\n", page_total); +} + +void mi_debug_show_arenas(void) mi_attr_noexcept { + mi_debug_show_arenas_ex(mi_heap_main(), true /* show pages */, true /* narrow? */); +} + +void mi_arenas_print(void) mi_attr_noexcept { + mi_debug_show_arenas(); +} + + +/* ----------------------------------------------------------- + Reserve a huge page arena. +----------------------------------------------------------- */ +// reserve at a specific numa node +int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept { + if (arena_id != NULL) *arena_id = NULL; + if (pages==0) return 0; + if (numa_node < -1) numa_node = -1; + if (numa_node >= 0) numa_node = numa_node % _mi_os_numa_node_count(); + mi_subproc_t* subproc = _mi_subproc(); + size_t hsize = 0; + size_t pages_reserved = 0; + mi_memid_t memid; + void* p = _mi_os_alloc_huge_os_pages(subproc, pages, numa_node, timeout_msecs, &pages_reserved, &hsize, &memid); + if (p==NULL || pages_reserved==0) { + _mi_warning_message("failed to reserve %zu GiB huge pages\n", pages); + return ENOMEM; + } + _mi_verbose_message("numa node %i: reserved %zu GiB huge pages (of the %zu GiB requested)\n", numa_node, pages_reserved, pages); + + if (!mi_manage_os_memory_ex2(subproc, p, hsize, numa_node, exclusive, memid, NULL, NULL, arena_id)) { + _mi_os_free(subproc, p, hsize, memid); + return ENOMEM; + } + return 0; +} + +int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept { + return mi_reserve_huge_os_pages_at_ex(pages, numa_node, timeout_msecs, false, NULL); +} + +// reserve huge pages evenly among the given number of numa nodes (or use the available ones as detected) +int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept { + if (pages == 0) return 0; + + // pages per numa node + int numa_count = (numa_nodes > 0 && numa_nodes <= INT_MAX ? (int)numa_nodes : _mi_os_numa_node_count()); + if (numa_count <= 0) { numa_count = 1; } + const size_t pages_per = pages / numa_count; + const size_t pages_mod = pages % numa_count; + const size_t timeout_per = (timeout_msecs==0 ? 0 : (timeout_msecs / numa_count) + 50); + + // reserve evenly among numa nodes + for (int numa_node = 0; numa_node < numa_count && pages > 0; numa_node++) { + size_t node_pages = pages_per; // can be 0 + if ((size_t)numa_node < pages_mod) { node_pages++; } + int err = mi_reserve_huge_os_pages_at(node_pages, numa_node, timeout_per); + if (err) return err; + if (pages < node_pages) { + pages = 0; + } + else { + pages -= node_pages; + } + } + + return 0; +} + +int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept { + MI_UNUSED(max_secs); + _mi_warning_message("mi_reserve_huge_os_pages is deprecated: use mi_reserve_huge_os_pages_interleave/at instead\n"); + if (pages_reserved != NULL) *pages_reserved = 0; + int err = mi_reserve_huge_os_pages_interleave(pages, 0, (size_t)(max_secs * 1000.0)); + if (err==0 && pages_reserved!=NULL) *pages_reserved = pages; + return err; +} + + +/* ----------------------------------------------------------- + Unloading and reloading an arena. +----------------------------------------------------------- */ +/* +static bool mi_arena_page_register(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) { + MI_UNUSED(arg); MI_UNUSED(slice_count); + mi_assert_internal(slice_count == 1); + mi_page_t* page = mi_arena_page_at_slice(arena, slice_index); + mi_assert_internal(mi_bitmap_is_setN(page->memid.mem.arena.arena->pages, page->memid.mem.arena.slice_index, 1)); + if (!_mi_page_map_register(page)) return false; // break + mi_assert_internal(mi_ptr_page(page)==page); + return true; +} + +mi_decl_nodiscard static bool mi_arena_pages_reregister(mi_arena_t* arena) { + return _mi_bitmap_forall_set(arena->pages, &mi_arena_page_register, arena, NULL); +} + +mi_decl_export bool mi_arena_unload(mi_arena_id_t arena_id, void** base, size_t* accessed_size, size_t* full_size) { + mi_arena_t* arena = _mi_arena_from_id(arena_id); + if (arena==NULL) { + return false; + } + else if (!arena->is_exclusive) { + _mi_warning_message("cannot unload a non-exclusive arena (id %zu at %p)\n", arena_id, arena); + return false; + } + else if (arena->memid.memkind != MI_MEM_EXTERNAL) { + _mi_warning_message("can only unload managed arena's for external memory (id %zu at %p)\n", arena_id, arena); + return false; + } + + // find accessed size + const size_t asize = _mi_size_of_slices(mi_arena_used_slices(arena)); + if (base != NULL) { *base = (void*)arena; } + if (full_size != NULL) { *full_size = arena->memid.mem.os.size; } + if (accessed_size != NULL) { *accessed_size = asize; } + + // adjust abandoned page count + mi_subproc_t* const subproc = arena->subproc; + for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) { + const size_t count = mi_bitmap_popcount(arena->pages_abandoned[bin]); + if (count > 0) { mi_atomic_decrement_acq_rel(&subproc->abandoned_count[bin]); } + } + + // unregister the pages + _mi_page_map_unregister_range(arena, asize); + + // set arena entry to NULL + const size_t count = mi_arenas_get_count(subproc); + for(size_t i = 0; i < count; i++) { + if (mi_arena_from_index(subproc, i) == arena) { + mi_atomic_store_ptr_release(mi_arena_t, &subproc->arenas[i], NULL); + if (i + 1 == count) { // try adjust the count? + size_t expected = count; + mi_atomic_cas_strong_acq_rel(&subproc->arena_count, &expected, count-1); + } + break; + } + } + return true; +} + +mi_decl_export bool mi_arena_reload(void* start, size_t size, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) { + // assume the memory area is already containing the arena + if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); } + if (start == NULL || size == 0) return false; + mi_arena_t* arena = (mi_arena_t*)start; + mi_memid_t memid = arena->memid; + if (memid.memkind != MI_MEM_EXTERNAL) { + _mi_warning_message("can only reload arena's from external memory (%p)\n", arena); + return false; + } + if (memid.mem.os.base != start) { + _mi_warning_message("the reloaded arena base address differs from the external memory (arena: %p, external: %p)\n", arena, start); + return false; + } + if (memid.mem.os.size != size) { + _mi_warning_message("the reloaded arena size differs from the external memory (arena size: %zu, external size: %zu)\n", arena->memid.mem.os.size, size); + return false; + } + if (!arena->is_exclusive) { + _mi_warning_message("the reloaded arena is not exclusive\n"); + return false; + } + + // re-initialize + arena->is_exclusive = true; + arena->commit_fun = commit_fun; + arena->commit_fun_arg = commit_fun_arg; + arena->subproc = _mi_subproc(); + if (!mi_arenas_add(arena->subproc, arena, arena_id)) { + return false; + } + if (!mi_arena_pages_reregister(arena)) { + // todo: clear arena entry in the subproc? + return false; + } + + // adjust abandoned page count + for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) { + const size_t count = mi_bitmap_popcount(arena->pages_abandoned[bin]); + if (count > 0) { mi_atomic_decrement_acq_rel(&arena->subproc->abandoned_count[bin]); } + } + + return true; +} + +*/ diff --git a/system/lib/mimalloc/src/arena/arena.h b/system/lib/mimalloc/src/arena/arena.h new file mode 100644 index 0000000000000..2e01fa4c84d25 --- /dev/null +++ b/system/lib/mimalloc/src/arena/arena.h @@ -0,0 +1,87 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#pragma once +#ifndef MI_ARENA_H +#define MI_ARENA_H + +#include "mimalloc/util/libc.h" +#include "mimalloc/arena/arena.h" +#include "bitmap.h" + +bool _mi_arena_reserve(mi_subproc_t* subproc, size_t req_size, bool allow_large, mi_arena_id_t* arena_id); +mi_decl_nodiscard bool _mi_arena_commit(mi_subproc_t* subproc, mi_arena_t* arena, void* start, size_t size, bool* is_zero, size_t stat_already_committed); +mi_decl_noinline void* _mi_arenas_try_alloc(mi_heap_t* heap, size_t slice_count, size_t alignment, bool commit, bool allow_large, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid); +void* _mi_arena_os_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid); + +// Check if an arena is suitable for a request arena +static inline bool mi_arena_is_suitable(mi_arena_t* arena, mi_arena_t* req_arena) { + if (arena == req_arena) return true; // they match + if (arena == NULL) return false; + if (req_arena == NULL && !arena->is_exclusive) return true; // or the arena is not exclusive, and we didn't request a specific one + if (arena->parent != NULL && arena->parent == req_arena) return true; // sub-arena? (note that req_arena is never a sub arena) + return false; +} + +// Get the number of arenas in a subproc +static inline size_t mi_arenas_get_count(mi_subproc_t* subproc) { + return mi_atomic_load_relaxed(&subproc->arena_count); +} + +static inline mi_arena_t* mi_arena_from_index(mi_subproc_t* subproc, size_t idx) { + mi_assert_internal(idx < mi_arenas_get_count(subproc)); + return mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[idx]); +} + +static inline size_t mi_arena_info_slices(mi_arena_t* arena) { + return arena->info_slices; +} + +// slices reserved for page meta info at the start of aligned chunks +static inline size_t mi_arena_page_meta_aligned_slice_count(void) { + #if MI_PAGE_META_IS_ALIGNED + return mi_divide_up(MI_PAGE_META_ALIGNED_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE); + #else + return 0; + #endif +} + +// Start of the arena memory area +static inline uint8_t* mi_arena_start(mi_arena_t* arena) { + return ((uint8_t*)arena->start); +} + +// Start of a slice +static inline uint8_t* mi_arena_slice_start(mi_arena_t* arena, size_t slice_index) { + mi_assert_internal(slice_index < arena->slice_count); + return (mi_arena_start(arena) + _mi_size_of_slices(slice_index)); +} + +// get the arena and slice span +static inline mi_arena_t* mi_arena_from_memid(mi_memid_t memid, size_t* slice_index, size_t* slice_count) { + mi_assert_internal(memid.memkind == MI_MEM_ARENA); + mi_arena_t* arena = memid.mem.arena.arena; + if (slice_index!=NULL) { *slice_index = memid.mem.arena.slice_index; } + if (slice_count!=NULL) { *slice_count = memid.mem.arena.slice_count; } + return arena; +} + +// initialize the bitmap for `slice_count` slices at base (and advance the base pointer) +static inline mi_bitmap_t* mi_arena_bitmap_init(size_t slice_count, uint8_t** base) { + mi_bitmap_t* bitmap = (mi_bitmap_t*)(*base); + *base = (*base) + mi_bitmap_init(bitmap, slice_count, true /* already zero */); + return bitmap; +} + +typedef bool (mi_forall_arena_fun_t)(mi_arena_t* arena, const void* arg, void** result); + +bool mi_forall_arenas(mi_heap_t* heap, mi_arena_t* req_arena, size_t tseq, mi_forall_arena_fun_t* visit, const void* arg, void** result); +bool mi_forall_suitable_arenas(mi_heap_t* heap, mi_arena_t* req_arena, size_t tseq, bool match_numa, int numa_node, + bool allow_large, mi_forall_arena_fun_t* visit, const void* arg, void** result); + + +#endif diff --git a/system/lib/mimalloc/src/bitmap.c b/system/lib/mimalloc/src/arena/bitmap-chunk.c similarity index 51% rename from system/lib/mimalloc/src/bitmap.c rename to system/lib/mimalloc/src/arena/bitmap-chunk.c index 238c4d4c4c12a..32122ff1ab871 100644 --- a/system/lib/mimalloc/src/bitmap.c +++ b/system/lib/mimalloc/src/arena/bitmap-chunk.c @@ -10,10 +10,10 @@ Concurrent bitmap that can set/reset sequences of bits atomically ---------------------------------------------------------------------------- */ #include "mimalloc.h" -#include "mimalloc/internal.h" -#include "mimalloc/bits.h" #include "mimalloc/prim.h" // _mi_prim_thread_yield -#include "bitmap.h" +#include "mimalloc/util/libc.h" +#include "mimalloc/util/stats.h" +#include "bitmap-chunk.h" #ifndef MI_OPT_SIMD #define MI_OPT_SIMD 0 @@ -31,32 +31,32 @@ static inline size_t mi_bfield_clz(mi_bfield_t x) { return mi_clz(x); } -static inline size_t mi_bfield_popcount(mi_bfield_t x) { +size_t mi_bfield_popcount(mi_bfield_t x) { return mi_popcount(x); } -static inline mi_bfield_t mi_bfield_clear_least_bit(mi_bfield_t x) { +mi_bfield_t mi_bfield_clear_least_bit(mi_bfield_t x) { return (x & (x-1)); } // find the least significant bit that is set (i.e. count trailing zero's) // return false if `x==0` (with `*idx` undefined) and true otherwise, // with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`). -static inline bool mi_bfield_find_least_bit(mi_bfield_t x, size_t* idx) { +bool mi_bfield_find_least_bit(mi_bfield_t x, size_t* idx) { return mi_bsf(x,idx); } // find the most significant bit that is set. // return false if `x==0` (with `*idx` undefined) and true otherwise, // with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`). -static inline bool mi_bfield_find_highest_bit(mi_bfield_t x, size_t* idx) { +bool mi_bfield_find_highest_bit(mi_bfield_t x, size_t* idx) { return mi_bsr(x, idx); } // find each set bit in a bit field `x` and clear it, until it becomes zero. -static inline bool mi_bfield_foreach_bit(mi_bfield_t* x, size_t* idx) { +bool mi_bfield_foreach_bit(mi_bfield_t* x, size_t* idx) { const bool found = mi_bfield_find_least_bit(*x, idx); *x = mi_bfield_clear_least_bit(*x); return found; @@ -75,7 +75,7 @@ static inline mi_bfield_t mi_bfield_all_set(void) { } // mask of `bit_count` bits set shifted to the left by `shiftl` -static inline mi_bfield_t mi_bfield_mask(size_t bit_count, size_t shiftl) { +mi_bfield_t mi_bfield_mask(size_t bit_count, size_t shiftl) { mi_assert_internal(bit_count > 0); mi_assert_internal(bit_count + shiftl <= MI_BFIELD_BITS); mi_assert_internal(shiftl < MI_BFIELD_BITS); @@ -120,7 +120,7 @@ static inline void mi_bfield_atomic_clear_once_set(mi_subproc_t* subproc, _Atomi mi_subproc_stat_counter_increase(subproc, pages_unabandon_busy_wait, 1); } while ((old&mask)==0) { // busy wait - _mi_prim_thread_yield(); + _mi_prim_thread_yield(); old = mi_atomic_load_acquire(b); } } @@ -142,7 +142,7 @@ static inline bool mi_bfield_atomic_set_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t // `all_clear` is set to `true` if the new bfield became zero. static inline bool mi_bfield_atomic_clear_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, bool* all_clear) { mi_assert_internal(mask != 0); - const mi_bfield_t old = mi_atomic_and_acq_rel(b,~mask); + const mi_bfield_t old = mi_atomic_and_acq_rel(b,~mask); if (all_clear != NULL) { *all_clear = ((old&~mask)==0); } return ((old&mask) == mask); } @@ -165,7 +165,7 @@ static inline bool mi_bfield_atomic_try_clear_mask_optimistic(_Atomic(mi_bfield_ // note: we could also use a strong cas but generally an optimistic atomic and/or is more efficient (at least on arm64) // try to atomically clear the mask bits mi_bfield_t old = mi_atomic_and_acq_rel(b,~mask); - if (previous!=NULL) { *previous = old; } + if (previous!=NULL) { *previous = old; } if mi_likely((old&mask)==mask) { // all bits in the mask transitioned from 1 to 0 atomically return true; @@ -175,13 +175,13 @@ static inline bool mi_bfield_atomic_try_clear_mask_optimistic(_Atomic(mi_bfield_ if ((old&mask)!=0) { // restore accidentally cleared ones mi_atomic_or_acq_rel(b, old&mask); - if (did_temp_clear_bits!=NULL) { *did_temp_clear_bits = true; } + if (did_temp_clear_bits!=NULL) { *did_temp_clear_bits = true; } } return false; } } -// Tries to clear a bit atomically. For performance, it assumes there is a good chance of success. +// Tries to clear a bit atomically. For performance, it assumes there is a good chance of success. // Returns `true` if the bit transitioned from 1 to 0 // and `false` otherwise (leaving the bfield `b` as-is). // `all_clear` is set to true if the new bfield became zero (and false otherwise) @@ -191,7 +191,7 @@ mi_decl_maybe_unused static inline bool mi_bfield_atomic_try_clear_optimistic(_A return mi_bfield_atomic_try_clear_mask_optimistic(b, mask, previous, NULL); // single bit never clears temporarily } -// Tries to clear a byte atomically. For performance, it assumes there is a good chance of success. +// Tries to clear a byte atomically. For performance, it assumes there is a good chance of success. // Returns true if the byte atomically transitioned from 0xFF to 0 // `all_clear` is set to true if the new bfield became zero (and false otherwise) mi_decl_maybe_unused static inline bool mi_bfield_atomic_try_clear8_optimistic(_Atomic(mi_bfield_t)*b, size_t idx, mi_bfield_t* previous, bool* did_temp_clear_bits) { @@ -201,14 +201,14 @@ mi_decl_maybe_unused static inline bool mi_bfield_atomic_try_clear8_optimistic(_ return mi_bfield_atomic_try_clear_mask_optimistic(b, mask, previous, did_temp_clear_bits); } -// Try to clear a mask of bits atomically. For performance, it assumes these is a good chance of success. +// Try to clear a mask of bits atomically. For performance, it assumes these is a good chance of success. // Returns true all bits transitioned from all 1's to 0's. // and false otherwise leaving the bit field as-is. `all_clear` is set to true if the new bfield became zero. static inline bool mi_bfield_atomic_try_clear_mask(_Atomic(mi_bfield_t)* b, mi_bfield_t mask, bool* all_clear, bool* did_temp_clear_bits) { mi_assert_internal(mask != 0); mi_bfield_t old = mi_atomic_load_relaxed(b); // check first if there is a good chance the mask bits are all set bool cleared = false; - if ((old&mask)==mask) { + if ((old&mask)==mask) { cleared = mi_bfield_atomic_try_clear_mask_optimistic(b, mask, &old, did_temp_clear_bits); } if (all_clear != NULL) { *all_clear = ((cleared ? old&~mask : old) == 0); } @@ -284,7 +284,7 @@ static inline size_t mi_bfield_atomic_popcount_mask(_Atomic(mi_bfield_t)*b, mi_b // ------- mi_bchunk_set --------------------------------------- // Set a single bit -static inline bool mi_bchunk_set(mi_bchunk_t* chunk, size_t cidx, size_t* already_set) { +bool mi_bchunk_set(mi_bchunk_t* chunk, size_t cidx, size_t* already_set) { mi_assert_internal(cidx < MI_BCHUNK_BITS); const size_t i = cidx / MI_BFIELD_BITS; const size_t idx = cidx % MI_BFIELD_BITS; @@ -358,7 +358,7 @@ mi_decl_noinline static bool mi_bchunk_xsetNC(mi_xset_t set, mi_bchunk_t* chunk, return all_transition; } -static inline bool mi_bchunk_setN(mi_bchunk_t* chunk, size_t cidx, size_t n, size_t* already_set) { +bool mi_bchunk_setN(mi_bchunk_t* chunk, size_t cidx, size_t n, size_t* already_set) { mi_assert_internal(n>0 && n <= MI_BCHUNK_BITS); if (n==1) return mi_bchunk_set(chunk, cidx, already_set); // if (n==8 && (cidx%8) == 0) return mi_bchunk_set8(chunk, cidx, already_set); @@ -369,14 +369,14 @@ static inline bool mi_bchunk_setN(mi_bchunk_t* chunk, size_t cidx, size_t n, siz // ------- mi_bchunk_clear --------------------------------------- -static inline bool mi_bchunk_clear(mi_bchunk_t* chunk, size_t cidx, bool* all_clear) { +bool mi_bchunk_clear(mi_bchunk_t* chunk, size_t cidx, bool* all_clear) { mi_assert_internal(cidx < MI_BCHUNK_BITS); const size_t i = cidx / MI_BFIELD_BITS; const size_t idx = cidx % MI_BFIELD_BITS; return mi_bfield_atomic_clear(&chunk->bfields[i], idx, all_clear); } -static inline bool mi_bchunk_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* maybe_all_clear) { +bool mi_bchunk_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* maybe_all_clear) { mi_assert_internal(n>0 && n <= MI_BCHUNK_BITS); if (n==1) return mi_bchunk_clear(chunk, cidx, maybe_all_clear); // if (n==8) return mi_bchunk_clear8(chunk, cidx, maybe_all_clear); @@ -406,7 +406,7 @@ mi_decl_noinline static size_t mi_bchunk_popcountNC(mi_bchunk_t* chunk, size_t f } // Count set bits a sequence of `n` bits. -static inline size_t mi_bchunk_popcountN(mi_bchunk_t* chunk, size_t cidx, size_t n) { +size_t mi_bchunk_popcountN(mi_bchunk_t* chunk, size_t cidx, size_t n) { mi_assert_internal(cidx + n <= MI_BCHUNK_BITS); mi_assert_internal(n>0); if (n==0) return 0; @@ -442,7 +442,7 @@ mi_decl_noinline static bool mi_bchunk_is_xsetNC(mi_xset_t set, const mi_bchunk_ } // Check if a sequence of `n` bits within a chunk are all set/cleared. -static inline bool mi_bchunk_is_xsetN(mi_xset_t set, const mi_bchunk_t* chunk, size_t cidx, size_t n) { +bool mi_bchunk_is_xsetN(mi_xset_t set, const mi_bchunk_t* chunk, size_t cidx, size_t n) { mi_assert_internal(cidx + n <= MI_BCHUNK_BITS); mi_assert_internal(n>0); if (n==0) return true; @@ -565,7 +565,7 @@ mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t ci } -static inline bool mi_bchunk_try_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* maybe_all_clear, bool* did_temp_clear_bits) { +bool mi_bchunk_try_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* maybe_all_clear, bool* did_temp_clear_bits) { mi_assert_internal(n>0); // if (n==MI_BFIELD_BITS) return mi_bchunk_try_clearX(chunk, cidx, maybe_all_clear); if (n<=MI_BFIELD_BITS) return mi_bchunk_try_clearNX(chunk, cidx, n, maybe_all_clear, did_temp_clear_bits); @@ -597,7 +597,7 @@ static inline bool mi_bchunk_try_find_and_clear_at(mi_bchunk_t* chunk, size_t ch mi_bfield_t b = mi_atomic_load_relaxed(bfield); if (b==0) return false; int tries = 0; - do { + do { const mi_bfield_t mask = (b & (~b+1)); // == (b & -b) but avoids a compiler warning -- clear all bits except the least-significant one b = mi_atomic_and_acq_rel(bfield,~mask); // clear the bit and set `b` to the previous value if mi_likely((b&mask)==mask) { // if we transitioned from 1 to 0, we actually cleared it @@ -606,7 +606,7 @@ static inline bool mi_bchunk_try_find_and_clear_at(mi_bchunk_t* chunk, size_t ch *pidx = (chunk_idx*MI_BFIELD_BITS) + bitidx; mi_assert_internal(*pidx < MI_BCHUNK_BITS); return true; - } + } } while (b!=0 && ++tries <= 4); // limit tries to reduce possible contention return false; } @@ -615,8 +615,8 @@ static inline bool mi_bchunk_try_find_and_clear_at(mi_bchunk_t* chunk, size_t ch // set `*pidx` to the bit index (0 <= *pidx < MI_BCHUNK_BITS) on success. // This is used to find free slices and abandoned pages and should be efficient. // todo: try neon version -static inline bool mi_bchunk_try_find_and_clear(mi_bchunk_t* chunk, size_t* pidx) { - #if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==256) +bool mi_bchunk_try_find_and_clear(mi_bchunk_t* chunk, size_t* pidx) { + #if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==256) && (MI_BFIELD_SIZE==8) for(int tries=0; tries<4; tries++) { // paranoia: at most 4 tries const __m256i vec = _mm256_load_si256((const __m256i*)chunk->bfields); const __m256i vcmp = _mm256_cmpeq_epi64(vec, mi_mm256_zero()); // (elem64 == 0 ? 0xFF : 0) @@ -633,7 +633,7 @@ static inline bool mi_bchunk_try_find_and_clear(mi_bchunk_t* chunk, size_t* pidx __asm __volatile ("" : : "g"(chunk) : "memory"); #endif } - #elif MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512) + #elif MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512) && (MI_BFIELD_SIZE==8) for(int tries=0; tries<4; tries++) { // paranoia: at most 4 tries size_t chunk_idx = 0; #if 0 @@ -702,22 +702,22 @@ static inline bool mi_bchunk_try_find_and_clear(mi_bchunk_t* chunk, size_t* pidx if (mi_bchunk_try_find_and_clear_at(chunk, i, pidx)) return true; } #endif - return false; + return false; } -static inline bool mi_bchunk_try_find_and_clear_1(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) { +bool mi_bchunk_try_find_and_clear_1(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) { mi_assert_internal(n==1); MI_UNUSED(n); MI_UNUSED(did_temp_clear_bits); return mi_bchunk_try_find_and_clear(chunk, pidx); } mi_decl_maybe_unused static inline bool mi_bchunk_try_find_and_clear8_at(mi_bchunk_t* chunk, size_t chunk_idx, size_t* pidx, bool* did_temp_clear_bits) { - _Atomic(mi_bfield_t)* const bfield = &chunk->bfields[chunk_idx]; + _Atomic(mi_bfield_t)* const bfield = &chunk->bfields[chunk_idx]; mi_bfield_t b = mi_atomic_load_relaxed(bfield); if (b==0) return false; int tries = 0; do { // has_set8 has low bit in each byte set if the byte in x == 0xFF - const mi_bfield_t has_set8 = + const mi_bfield_t has_set8 = ((~b - MI_BFIELD_LO_BIT8) & // high bit set if byte in x is 0xFF or < 0x7F (b & MI_BFIELD_HI_BIT8)) // high bit set if byte in x is >= 0x80 >> 7; // shift high bit to low bit @@ -780,7 +780,7 @@ static mi_decl_noinline bool mi_bchunk_try_find_and_clear8(mi_bchunk_t* chunk, s return false; } -static inline bool mi_bchunk_try_find_and_clear_8(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) { +bool mi_bchunk_try_find_and_clear_8(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) { mi_assert_internal(n==8); MI_UNUSED(n); MI_UNUSED(did_temp_clear_bits); return mi_bchunk_try_find_and_clear8(chunk, pidx, did_temp_clear_bits); } @@ -790,7 +790,7 @@ static inline bool mi_bchunk_try_find_and_clear_8(mi_bchunk_t* chunk, size_t n, // and try to clear them atomically. // set `*pidx` to its bit index (0 <= *pidx <= MI_BCHUNK_BITS - n) on success. // will cross bfield boundaries. -mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) { +mi_decl_noinline bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) { if (n == 0 || n > MI_BFIELD_BITS) return false; const mi_bfield_t mask = mi_bfield_mask(n, 0); // for all fields in the chunk @@ -806,7 +806,7 @@ mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, const size_t bmask = mask<>idx == mask); if ((b&bmask) == bmask) { // found a match with all bits set, try clearing atomically - if mi_likely(mi_bfield_atomic_try_clear_mask_optimistic(&chunk->bfields[i], bmask, &b0, did_temp_clear_bits)) { // updates b0 with the previous value + if mi_likely(mi_bfield_atomic_try_clear_mask_optimistic(&chunk->bfields[i], bmask, &b0, did_temp_clear_bits)) { // updates b0 with the previous value *pidx = (i*MI_BFIELD_BITS) + idx; mi_assert_internal(*pidx < MI_BCHUNK_BITS); mi_assert_internal(*pidx + n <= MI_BCHUNK_BITS); @@ -852,7 +852,7 @@ mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, // and try to clear them atomically. // set `*pidx` to its bit index (0 <= *pidx <= MI_BCHUNK_BITS - n) on success. // This can cross bfield boundaries. -static mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) { +mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) { if (n == 0 || n > MI_BCHUNK_BITS) return false; // cannot be more than a chunk // we first scan ahead to see if there is a range of `n` set bits, and only then try to clear atomically @@ -919,7 +919,7 @@ static mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk, // ------- mi_bchunk_clear_once_set --------------------------------------- -static inline void mi_bchunk_clear_once_set(mi_subproc_t* subproc, mi_bchunk_t* chunk, size_t cidx) { +void mi_bchunk_clear_once_set(mi_subproc_t* subproc, mi_bchunk_t* chunk, size_t cidx) { mi_assert_internal(cidx < MI_BCHUNK_BITS); const size_t i = cidx / MI_BFIELD_BITS; const size_t idx = cidx % MI_BFIELD_BITS; @@ -931,7 +931,7 @@ static inline void mi_bchunk_clear_once_set(mi_subproc_t* subproc, mi_bchunk_t* // are all bits in a bitmap chunk clear? -static inline bool mi_bchunk_all_are_clear_relaxed(mi_bchunk_t* chunk) { +bool mi_bchunk_all_are_clear_relaxed(mi_bchunk_t* chunk) { #if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==256) const __m256i vec = _mm256_load_si256((const __m256i*)chunk->bfields); return mi_mm256_is_zero(vec); @@ -956,7 +956,7 @@ static inline bool mi_bchunk_all_are_clear_relaxed(mi_bchunk_t* chunk) { } // are all bits in a bitmap chunk set? -static inline bool mi_bchunk_all_are_set_relaxed(mi_bchunk_t* chunk) { +bool mi_bchunk_all_are_set_relaxed(mi_bchunk_t* chunk) { #if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==256) const __m256i vec = _mm256_load_si256((const __m256i*)chunk->bfields); return mi_mm256_is_ones(vec); @@ -981,7 +981,7 @@ static inline bool mi_bchunk_all_are_set_relaxed(mi_bchunk_t* chunk) { } -static bool mi_bchunk_bsr(mi_bchunk_t* chunk, size_t* pidx) { +bool mi_bchunk_bsr(mi_bchunk_t* chunk, size_t* pidx) { for (size_t i = MI_BCHUNK_FIELDS; i > 0; ) { i--; mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]); @@ -994,7 +994,7 @@ static bool mi_bchunk_bsr(mi_bchunk_t* chunk, size_t* pidx) { return false; } -static bool mi_bchunk_bsr_inv(mi_bchunk_t* chunk, size_t* pidx) { +bool mi_bchunk_bsr_inv(mi_bchunk_t* chunk, size_t* pidx) { for (size_t i = MI_BCHUNK_FIELDS; i > 0; ) { i--; mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]); @@ -1007,7 +1007,7 @@ static bool mi_bchunk_bsr_inv(mi_bchunk_t* chunk, size_t* pidx) { return false; } -static size_t mi_bchunk_popcount(mi_bchunk_t* chunk) { +size_t mi_bchunk_popcount(mi_bchunk_t* chunk) { size_t popcount = 0; for (size_t i = 0; i < MI_BCHUNK_FIELDS; i++) { const mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]); @@ -1017,71 +1017,14 @@ static size_t mi_bchunk_popcount(mi_bchunk_t* chunk) { } -/* -------------------------------------------------------------------------------- - bitmap chunkmap --------------------------------------------------------------------------------- */ - -static void mi_bitmap_chunkmap_set(mi_bitmap_t* bitmap, size_t chunk_idx) { - mi_assert(chunk_idx < mi_bitmap_chunk_count(bitmap)); - mi_bchunk_set(&bitmap->chunkmap, chunk_idx, NULL); -} - -static bool mi_bitmap_chunkmap_try_clear(mi_bitmap_t* bitmap, size_t chunk_idx) { - mi_assert(chunk_idx < mi_bitmap_chunk_count(bitmap)); - // check if the corresponding chunk is all clear - if (!mi_bchunk_all_are_clear_relaxed(&bitmap->chunks[chunk_idx])) return false; - // clear the chunkmap bit - mi_bchunk_clear(&bitmap->chunkmap, chunk_idx, NULL); - // .. but a concurrent set may have happened in between our all-clear test and the clearing of the - // bit in the mask. We check again to catch this situation. - if (!mi_bchunk_all_are_clear_relaxed(&bitmap->chunks[chunk_idx])) { - mi_bchunk_set(&bitmap->chunkmap, chunk_idx, NULL); - return false; - } - return true; -} - - -/* -------------------------------------------------------------------------------- - bitmap --------------------------------------------------------------------------------- */ - -size_t mi_bitmap_size(size_t bit_count, size_t* pchunk_count) { - mi_assert_internal((bit_count % MI_BCHUNK_BITS) == 0); - bit_count = _mi_align_up(bit_count, MI_BCHUNK_BITS); - mi_assert_internal(bit_count <= MI_BITMAP_MAX_BIT_COUNT); - mi_assert_internal(bit_count > 0); - const size_t chunk_count = bit_count / MI_BCHUNK_BITS; - mi_assert_internal(chunk_count >= 1); - const size_t size = offsetof(mi_bitmap_t,chunks) + (chunk_count * MI_BCHUNK_SIZE); - mi_assert_internal( (size%MI_BCHUNK_SIZE) == 0 ); - if (pchunk_count != NULL) { *pchunk_count = chunk_count; } - return size; -} - - -// initialize a bitmap to all unset; avoid a mem_zero if `already_zero` is true -// returns the size of the bitmap -size_t mi_bitmap_init(mi_bitmap_t* bitmap, size_t bit_count, bool already_zero) { - size_t chunk_count; - const size_t size = mi_bitmap_size(bit_count, &chunk_count); - if (!already_zero) { - _mi_memzero_aligned(bitmap, size); - } - mi_atomic_store_release(&bitmap->chunk_count, chunk_count); - mi_assert_internal(mi_atomic_load_relaxed(&bitmap->chunk_count) <= MI_BITMAP_MAX_CHUNK_COUNT); - return size; -} - - // Set a sequence of `n` bits in the bitmap (and can cross chunks). Not atomic so only use if local to a thread. -static void mi_bchunks_unsafe_setN(mi_bchunk_t* chunks, mi_bchunkmap_t* cmap, size_t idx, size_t n) { +void mi_bchunks_unsafe_setN(mi_bchunk_t* chunks, mi_bchunkmap_t* cmap, size_t idx, size_t n) { mi_assert_internal(n>0); // start chunk and index size_t chunk_idx = idx / MI_BCHUNK_BITS; const size_t cidx = idx % MI_BCHUNK_BITS; - const size_t ccount = ((cidx + n - 1) / MI_BCHUNK_BITS) + 1; + const size_t ccount = ((cidx + n - 1) / MI_BCHUNK_BITS) + 1; // first update the chunkmap mi_bchunk_setN(cmap, chunk_idx, ccount, NULL); @@ -1096,7 +1039,7 @@ static void mi_bchunks_unsafe_setN(mi_bchunk_t* chunks, mi_bchunkmap_t* cmap, si n -= m; const size_t mid_chunks = n / MI_BCHUNK_BITS; if (mid_chunks > 0) { - _mi_memset(&chunks[chunk_idx], ~0, mid_chunks * MI_BCHUNK_SIZE); + mi_memset(&chunks[chunk_idx], ~0, mid_chunks * MI_BCHUNK_SIZE); chunk_idx += mid_chunks; n -= (mid_chunks * MI_BCHUNK_BITS); } @@ -1107,896 +1050,3 @@ static void mi_bchunks_unsafe_setN(mi_bchunk_t* chunks, mi_bchunkmap_t* cmap, si mi_bchunk_setN(&chunks[chunk_idx], 0, n, NULL); } } - -// Set a sequence of `n` bits in the bitmap (and can cross chunks). Not atomic so only use if local to a thread. -void mi_bitmap_unsafe_setN(mi_bitmap_t* bitmap, size_t idx, size_t n) { - mi_assert_internal(n>0); - mi_assert_internal(idx + n <= mi_bitmap_max_bits(bitmap)); - mi_bchunks_unsafe_setN(&bitmap->chunks[0], &bitmap->chunkmap, idx, n); -} - - - - -// ------- mi_bitmap_xset --------------------------------------- - -// Set a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from 0's to 1's (or 1's to 0's). -bool mi_bitmap_setN(mi_bitmap_t* bitmap, size_t idx, size_t n, size_t* palready_set) { - mi_assert_internal(n>0); - const size_t maxbits = mi_bitmap_max_bits(bitmap); - mi_assert_internal(idx + n <= maxbits); - if (idx+n > maxbits) { // paranoia - if (idx >= maxbits) return false; - n = maxbits - idx; - } - - // iterate through the chunks - size_t chunk_idx = idx / MI_BCHUNK_BITS; - size_t cidx = idx % MI_BCHUNK_BITS; - bool were_allclear = true; - size_t already_set = 0; - while (n > 0) { - const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n); - size_t _already_set = 0; - were_allclear = mi_bchunk_setN(&bitmap->chunks[chunk_idx], cidx, m, &_already_set) && were_allclear; - already_set += _already_set; - mi_bitmap_chunkmap_set(bitmap, chunk_idx); // set afterwards - mi_assert_internal(m <= n); - n -= m; - cidx = 0; - chunk_idx++; - } - if (palready_set != NULL) { *palready_set = already_set; } - return were_allclear; -} - -// Clear a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from 1's to 0's. -bool mi_bitmap_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n) { - mi_assert_internal(n>0); - const size_t maxbits = mi_bitmap_max_bits(bitmap); - mi_assert_internal(idx + n <= maxbits); - if (idx+n > maxbits) { // paranoia - if (idx >= maxbits) return false; - n = maxbits - idx; - } - - // iterate through the chunks - size_t chunk_idx = idx / MI_BCHUNK_BITS; - size_t cidx = idx % MI_BCHUNK_BITS; - bool were_allset = true; - while (n > 0) { - const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n); - bool maybe_all_clear = false; - were_allset = mi_bchunk_clearN(&bitmap->chunks[chunk_idx], cidx, m, &maybe_all_clear) && were_allset; - if (maybe_all_clear) { mi_bitmap_chunkmap_try_clear(bitmap, chunk_idx); } - mi_assert_internal(m <= n); - n -= m; - cidx = 0; - chunk_idx++; - } - return were_allset; -} - -// Count bits set in a range of `n` bits. -size_t mi_bitmap_popcountN( mi_bitmap_t* bitmap, size_t idx, size_t n) { - mi_assert_internal(n>0); - const size_t maxbits = mi_bitmap_max_bits(bitmap); - mi_assert_internal(idx + n <= maxbits); - if (idx+n > maxbits) { // paranoia - if (idx >= maxbits) return 0; - n = maxbits - idx; - } - - // iterate through the chunks - size_t chunk_idx = idx / MI_BCHUNK_BITS; - size_t cidx = idx % MI_BCHUNK_BITS; - size_t popcount = 0; - while (n > 0) { - const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n); - popcount += mi_bchunk_popcountN(&bitmap->chunks[chunk_idx], cidx, m); - mi_assert_internal(m <= n); - n -= m; - cidx = 0; - chunk_idx++; - } - return popcount; -} - - -// Set/clear a bit in the bitmap; returns `true` if atomically transitioned from 0 to 1 (or 1 to 0) -bool mi_bitmap_set(mi_bitmap_t* bitmap, size_t idx) { - return mi_bitmap_setN(bitmap, idx, 1, NULL); -} - -bool mi_bitmap_clear(mi_bitmap_t* bitmap, size_t idx) { - return mi_bitmap_clearN(bitmap, idx, 1); -} - - - -// ------- mi_bitmap_is_xset --------------------------------------- - -// Is a sequence of n bits already all set/cleared? -bool mi_bitmap_is_xsetN(mi_xset_t set, mi_bitmap_t* bitmap, size_t idx, size_t n) { - mi_assert_internal(n>0); - const size_t maxbits = mi_bitmap_max_bits(bitmap); - mi_assert_internal(idx + n <= maxbits); - if (idx+n > maxbits) { // paranoia - if (idx >= maxbits) return false; - n = maxbits - idx; - } - - // iterate through the chunks - size_t chunk_idx = idx / MI_BCHUNK_BITS; - size_t cidx = idx % MI_BCHUNK_BITS; - bool xset = true; - while (n > 0 && xset) { - const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n); - xset = mi_bchunk_is_xsetN(set, &bitmap->chunks[chunk_idx], cidx, m) && xset; - mi_assert_internal(m <= n); - n -= m; - cidx = 0; - chunk_idx++; - } - return xset; -} - -bool mi_bitmap_is_all_clear(mi_bitmap_t* bitmap) { - return mi_bitmap_is_xsetN(MI_BIT_CLEAR, bitmap, 0, mi_bitmap_max_bits(bitmap)); -} - -/* -------------------------------------------------------------------------------- - Iterate through a bfield --------------------------------------------------------------------------------- */ - -// Cycle iteration through a bitfield. This is used to space out threads -// so there is less chance of contention. When searching for a free page we -// like to first search only the accessed part (so we reuse better). This -// high point is called the `cycle`. -// -// We then iterate through the bitfield as: -// first: [start, cycle> -// then : [0, start> -// then : [cycle, MI_BFIELD_BITS> -// -// The start is determined usually as `tseq % cycle` to have each thread -// start at a different spot. -// - We use `popcount` to improve branch prediction (maybe not needed? can we simplify?) -// - The `cycle_mask` is the part `[start, cycle>`. -#define mi_bfield_iterate(bfield,start,cycle,name_idx,SUF) { \ - mi_assert_internal(start <= cycle); \ - mi_assert_internal(start < MI_BFIELD_BITS); \ - mi_assert_internal(cycle <= MI_BFIELD_BITS); \ - const mi_bfield_t _cycle_mask##SUF = mi_bfield_mask(cycle - start, start); \ - size_t _bcount##SUF = mi_bfield_popcount(bfield); \ - mi_bfield_t _b##SUF = bfield & _cycle_mask##SUF; /* process [start, cycle> first*/\ - while(_bcount##SUF > 0) { \ - _bcount##SUF--;\ - if (_b##SUF==0) { _b##SUF = bfield & ~_cycle_mask##SUF; } /* process [0,start> + [cycle, MI_BFIELD_BITS> next */ \ - /* size_t name_idx; */ \ - const bool _found##SUF = mi_bfield_find_least_bit(_b##SUF,&name_idx); \ - _b##SUF = mi_bfield_clear_least_bit(_b##SUF); /* clear early so `continue` works */ \ - mi_assert_internal(_found##SUF); MI_UNUSED(_found##SUF); \ - { \ - -#define mi_bfield_iterate_end(SUF) \ - } \ - } \ -} - - -#define mi_bfield_cycle_iterate(bfield,tseq,cycle,name_idx,SUF) { \ - const size_t _start##SUF = (uint32_t)(tseq) % (uint32_t)(cycle); /* or: 0 to always search from the start? */\ - mi_bfield_iterate(bfield,_start##SUF,cycle,name_idx,SUF) - -#define mi_bfield_cycle_iterate_end(SUF) \ - mi_bfield_iterate_end(SUF); \ -} - - -/* -------------------------------------------------------------------------------- - mi_bitmap_find - (used to find free pages) --------------------------------------------------------------------------------- */ - -typedef bool (mi_bitmap_visit_fun_t)(mi_bitmap_t* bitmap, size_t chunk_idx, size_t n, size_t* idx, void* arg1, void* arg2); - -// Go through the bitmap and for every sequence of `n` set bits, call the visitor function. -// If it returns `true` stop the search. -static inline bool mi_bitmap_find(mi_bitmap_t* bitmap, size_t tseq, size_t n, size_t* pidx, mi_bitmap_visit_fun_t* on_find, void* arg1, void* arg2) -{ - const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS); - for (size_t i = 0; i < chunkmap_max; i++) { - // and for each chunkmap entry we iterate over its bits to find the chunks - const mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]); - size_t hi; - if (mi_bfield_find_highest_bit(cmap_entry, &hi)) { - size_t eidx = 0; - mi_bfield_cycle_iterate(cmap_entry, tseq%8, hi+1, eidx, Y) // reduce the tseq to 8 bins to reduce using extra memory (see `mstress`) - { - mi_assert_internal(eidx <= MI_BFIELD_BITS); - const size_t chunk_idx = i*MI_BFIELD_BITS + eidx; - mi_assert_internal(chunk_idx < mi_bitmap_chunk_count(bitmap)); - if ((*on_find)(bitmap, chunk_idx, n, pidx, arg1, arg2)) { - return true; - } - } - mi_bfield_cycle_iterate_end(Y); - } - } - return false; -} - - -/* -------------------------------------------------------------------------------- - Bitmap: try_find_and_claim -- used to allocate abandoned pages - note: the compiler will fully inline the indirect function call --------------------------------------------------------------------------------- */ - -typedef struct mi_claim_fun_data_s { - mi_arena_t* arena; -} mi_claim_fun_data_t; - -static bool mi_bitmap_try_find_and_claim_visit(mi_bitmap_t* bitmap, size_t chunk_idx, size_t n, size_t* pidx, void* arg1, void* arg2) -{ - mi_assert_internal(n==1); MI_UNUSED(n); - mi_claim_fun_t* claim_fun = (mi_claim_fun_t*)arg1; - mi_claim_fun_data_t* claim_data = (mi_claim_fun_data_t*)arg2; - size_t cidx; - if mi_likely(mi_bchunk_try_find_and_clear(&bitmap->chunks[chunk_idx], &cidx)) { - const size_t slice_index = (chunk_idx * MI_BCHUNK_BITS) + cidx; - mi_assert_internal(slice_index < mi_bitmap_max_bits(bitmap)); - bool keep_set = true; - if ((*claim_fun)(slice_index, claim_data->arena, &keep_set)) { - // success! - mi_assert_internal(!keep_set); - *pidx = slice_index; - return true; - } - else { - // failed to claim it, set abandoned mapping again (unless the page was freed and keep_set will be false) - if (keep_set) { - const bool wasclear = mi_bchunk_set(&bitmap->chunks[chunk_idx], cidx, NULL); - mi_bitmap_chunkmap_set(bitmap, chunk_idx); - mi_assert_internal(wasclear); MI_UNUSED(wasclear); - } - } - } - else { - // we may find that all are cleared only on a second iteration but that is ok as - // the chunkmap is a conservative approximation. - mi_bitmap_chunkmap_try_clear(bitmap, chunk_idx); - } - return false; -} - -// Find a set bit in the bitmap and try to atomically clear it and claim it. -// (Used to find pages in the pages_abandoned bitmaps.) -mi_decl_nodiscard bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx, - mi_claim_fun_t* claim, mi_arena_t* arena ) -{ - mi_claim_fun_data_t claim_data = { arena }; - return mi_bitmap_find(bitmap, tseq, 1, pidx, &mi_bitmap_try_find_and_claim_visit, (void*)claim, &claim_data); -} - - -bool mi_bitmap_bsr(mi_bitmap_t* bitmap, size_t* idx) { - const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS); - for (size_t i = chunkmap_max; i > 0; ) { - i--; - mi_bfield_t cmap = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]); - size_t cmap_idx; - if (mi_bsr(cmap,&cmap_idx)) { - // from highest chunk to lowest (scan all in case the cmap entry was stale) - for (size_t j = cmap_idx+1; j>0; ) { - j--; - const size_t chunk_idx = (i*MI_BFIELD_BITS) + j; - size_t cidx; - if (mi_bchunk_bsr(&bitmap->chunks[chunk_idx], &cidx)) { - *idx = (chunk_idx * MI_BCHUNK_BITS) + cidx; - return true; - } - } - } - } - return false; -} - -// Return count of all set bits in a bitmap. -size_t mi_bitmap_popcount(mi_bitmap_t* bitmap) { - // for all chunkmap entries - size_t popcount = 0; - const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS); - for (size_t i = 0; i < chunkmap_max; i++) { - mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]); - size_t cmap_idx; - // for each chunk (corresponding to a set bit in a chunkmap entry) - while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) { - const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx; - // count bits in a chunk - popcount += mi_bchunk_popcount(&bitmap->chunks[chunk_idx]); - } - } - return popcount; -} - - - -// Clear a bit once it is set. -void mi_bitmap_clear_once_set(mi_subproc_t* subproc, mi_bitmap_t* bitmap, size_t idx) { - mi_assert_internal(idx < mi_bitmap_max_bits(bitmap)); - const size_t chunk_idx = idx / MI_BCHUNK_BITS; - const size_t cidx = idx % MI_BCHUNK_BITS; - mi_assert_internal(chunk_idx < mi_bitmap_chunk_count(bitmap)); - mi_bchunk_clear_once_set(subproc, &bitmap->chunks[chunk_idx], cidx); -} - - -// Visit all set bits in a bitmap. -// todo: optimize further? maybe use avx512 to directly get all indices using a mask_compressstore? -bool _mi_bitmap_forall_set(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg) { - // for all chunkmap entries - const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS); - for(size_t i = 0; i < chunkmap_max; i++) { - mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]); - size_t cmap_idx; - // for each chunk (corresponding to a set bit in a chunkmap entry) - while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) { - const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx; - // for each chunk field - mi_bchunk_t* const chunk = &bitmap->chunks[chunk_idx]; - for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) { - const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS); - mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[j]); - size_t bidx; - while (mi_bfield_foreach_bit(&b, &bidx)) { - const size_t idx = base_idx + bidx; - if (!visit(idx, 1, arena, arg)) return false; - } - } - } - } - return true; -} - -// Visit all set bits in a bitmap but try to return ranges (within bfields) if possible. -// Also clear those ranges atomically. -// Used by purging to purge larger ranges when possible -// todo: optimize further? maybe use avx512 to directly get all indices using a mask_compressstore? -bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg) { - // for all chunkmap entries - const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS); - for (size_t i = 0; i < chunkmap_max; i++) { - mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]); - size_t cmap_idx; - // for each chunk (corresponding to a set bit in a chunkmap entry) - while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) { - const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx; - // for each chunk field - mi_bchunk_t* const chunk = &bitmap->chunks[chunk_idx]; - for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) { - const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS); - mi_bfield_t b = mi_atomic_exchange_relaxed(&chunk->bfields[j], (mi_bfield_t)0); - #if MI_DEBUG > 1 - const size_t bpopcount = mi_popcount(b); - size_t rngcount = 0; - #endif - size_t bidx; - while (mi_bfield_find_least_bit(b, &bidx)) { - size_t rng = mi_ctz(~(b>>bidx)); // all the set bits from bidx - #if MI_DEBUG > 1 - rngcount += rng; - #endif - const size_t idx = base_idx + bidx; - mi_assert_internal(rng>=1 && rng<=MI_BFIELD_BITS); - mi_assert_internal((idx % MI_BFIELD_BITS) + rng <= MI_BFIELD_BITS); - mi_assert_internal((idx / MI_BCHUNK_BITS) < mi_bitmap_chunk_count(bitmap)); - // clear rng bits in b - b = b & ~mi_bfield_mask(rng, bidx); - if (!visit(idx, rng, arena, arg)) { - // break early: reset the non-visited bits - if (b!=0) { - mi_atomic_or_relaxed(&chunk->bfields[j], b); - } - return false; - } - } - mi_assert_internal(rngcount == bpopcount); - } - } - } - return true; -} - -// Visit all set bits in a bitmap but try to return ranges (within bfields) if possible, -// but only in chunks of at least `rngslices` slices (that are also aligned at `rngslices`) -// and clear those ranges atomically. -// However, the `rngslices` are capped at `MI_BFIELD_BITS` at most. -// Used by purging to purge larger ranges when possible. With transparent huge pages we only -// want to purge whole huge pages (2 MiB) at a time which is what the `rngslices` parameter achieves. -bool _mi_bitmap_forall_setc_rangesn(mi_bitmap_t* bitmap, size_t rngslices, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg) -{ - // use the generic routine for `rngslices<=1` (as that one finds longest ranges at a time) - if (rngslices<=1) { - return _mi_bitmap_forall_setc_ranges(bitmap, visit, arena, arg); - } - // mi_assert_internal(rngslices <= MI_BFIELD_BITS); - if (rngslices > MI_BFIELD_BITS) { rngslices = MI_BFIELD_BITS; } // cap at MI_BFIELD_BITS at most - - // for all chunkmap entries - const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS); - for (size_t i = 0; i < chunkmap_max; i++) { - mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]); - size_t cmap_idx; - // for each chunk (corresponding to a set bit in a chunkmap entry) - while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) { - const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx; - // for each chunk field - mi_bchunk_t* const chunk = &bitmap->chunks[chunk_idx]; - for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) { - const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS); - mi_bfield_t b = mi_atomic_exchange_relaxed(&chunk->bfields[j], (mi_bfield_t)0); // atomic clear - mi_bfield_t skipped = 0; // but track which bits we skip so we can restore them - size_t shift; - for(shift = 0; rngslices + shift <= MI_BFIELD_BITS; shift += rngslices) { // per `rngslices` to keep alignment - const mi_bfield_t rngmask = mi_bfield_mask(rngslices, shift); - if ((b & rngmask) == rngmask) { - const size_t idx = base_idx + shift; - if (!visit(idx, rngslices, arena, arg)) { - // break early: restore non-visited entries - mi_bfield_t notyet_visited = 0; - if (rngslices + shift < MI_BFIELD_BITS) { - notyet_visited = (b & (~(mi_bfield_t)0 << (shift + rngslices))); - } - mi_assert_internal((notyet_visited & skipped) == 0); - if ((notyet_visited | skipped) != 0) { - mi_atomic_or_relaxed(&chunk->bfields[j], notyet_visited | skipped); - } - return false; - } - } - else { - skipped = skipped | (b & rngmask); - } - } - if (shift < MI_BFIELD_BITS) { - // there are some non-visited top bits when `MI_BFIELD_BITS % rngslices != 0`. - mi_assert_internal(MI_BFIELD_BITS % rngslices != 0); - skipped = skipped | (b & (~(mi_bfield_t)0 << shift)); - } - if (skipped != 0) { - // restore non-visited entries - mi_atomic_or_relaxed(&chunk->bfields[j], skipped); - } - } - } - } - return true; -} - - -/* -------------------------------------------------------------------------------- - binned bitmap's --------------------------------------------------------------------------------- */ - - -size_t mi_bbitmap_size(size_t bit_count, size_t* pchunk_count) { - // mi_assert_internal((bit_count % MI_BCHUNK_BITS) == 0); - bit_count = _mi_align_up(bit_count, MI_BCHUNK_BITS); - mi_assert_internal(bit_count <= MI_BITMAP_MAX_BIT_COUNT); - mi_assert_internal(bit_count > 0); - const size_t chunk_count = bit_count / MI_BCHUNK_BITS; - mi_assert_internal(chunk_count >= 1); - const size_t size = offsetof(mi_bbitmap_t,chunks) + (chunk_count * MI_BCHUNK_SIZE); - mi_assert_internal( (size%MI_BCHUNK_SIZE) == 0 ); - if (pchunk_count != NULL) { *pchunk_count = chunk_count; } - return size; -} - -// initialize a bitmap to all unset; avoid a mem_zero if `already_zero` is true -// returns the size of the bitmap -size_t mi_bbitmap_init(mi_subproc_t* subproc, mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero) { - size_t chunk_count; - const size_t size = mi_bbitmap_size(bit_count, &chunk_count); - if (!already_zero) { - _mi_memzero_aligned(bbitmap, size); - } - mi_atomic_store_release(&bbitmap->chunk_count, chunk_count); - mi_assert_internal(mi_atomic_load_relaxed(&bbitmap->chunk_count) <= MI_BITMAP_MAX_CHUNK_COUNT); - bbitmap->subproc = subproc; - return size; -} - -void mi_bbitmap_unsafe_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) { - mi_assert_internal(n>0); - mi_assert_internal(idx + n <= mi_bbitmap_max_bits(bbitmap)); - mi_bchunks_unsafe_setN(&bbitmap->chunks[0], &bbitmap->chunkmap, idx, n); -} - -bool mi_bbitmap_bsr_inv(mi_bbitmap_t* bbitmap, size_t* idx) { - // scan for highest zero bit in the bitmap - // note: we cannot use the chunkmap since that only conservatively denotes if there might be a set bit in a chuck - // todo: bbitmap_init rounds up the bitcount to BCHUNK_BITS and we should skip the top-padding! - const size_t chunk_count = mi_bbitmap_chunk_count(bbitmap); - for(size_t i = chunk_count; i > 0; ) { - i--; - size_t cidx; - if (mi_bchunk_bsr_inv(&bbitmap->chunks[i], &cidx)) { - *idx = (i * MI_BCHUNK_BITS) + cidx; - return true; - } - } - return false; -} - - -/* -------------------------------------------------------------------------------- - binned bitmap used to track free slices --------------------------------------------------------------------------------- */ - -// Assign a specific size bin to a chunk -static void mi_bbitmap_set_chunk_bin(mi_bbitmap_t* bbitmap, size_t chunk_idx, mi_chunkbin_t bin) { - mi_assert_internal(chunk_idx < mi_bbitmap_chunk_count(bbitmap)); - for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin < MI_CBIN_NONE; ibin = mi_chunkbin_inc(ibin)) { - if (ibin == bin) { - const bool was_clear = mi_bchunk_set(& bbitmap->chunkmap_bins[ibin], chunk_idx, NULL); - if (was_clear) { mi_subproc_stat_increase(bbitmap->subproc, chunk_bins[ibin],1); } - } - else { - const bool was_set = mi_bchunk_clear(&bbitmap->chunkmap_bins[ibin], chunk_idx, NULL); - if (was_set) { mi_subproc_stat_decrease(bbitmap->subproc,chunk_bins[ibin],1); } - } - } -} - -mi_chunkbin_t mi_bbitmap_debug_get_bin(const mi_bchunkmap_t* chunkmap_bins, size_t chunk_idx) { - for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin < MI_CBIN_NONE; ibin = mi_chunkbin_inc(ibin)) { - if (mi_bchunk_is_xsetN(MI_BIT_SET, &chunkmap_bins[ibin], chunk_idx, 1)) { - return ibin; - } - } - return MI_CBIN_NONE; -} - -// Track the index of the highest chunk that is accessed. -static void mi_bbitmap_chunkmap_set_max(mi_bbitmap_t* bbitmap, size_t chunk_idx) { - size_t oldmax = mi_atomic_load_relaxed(&bbitmap->chunk_max_accessed); - if mi_unlikely(chunk_idx > oldmax) { - mi_atomic_cas_strong_relaxed(&bbitmap->chunk_max_accessed, &oldmax, chunk_idx); - } -} - -// Set a bit in the chunkmap -static void mi_bbitmap_chunkmap_set(mi_bbitmap_t* bbitmap, size_t chunk_idx, bool check_all_set) { - mi_assert(chunk_idx < mi_bbitmap_chunk_count(bbitmap)); - if (check_all_set) { - if (mi_bchunk_all_are_set_relaxed(&bbitmap->chunks[chunk_idx])) { - // all slices are free in this chunk: return back to the NONE bin - mi_bbitmap_set_chunk_bin(bbitmap, chunk_idx, MI_CBIN_NONE); - } - } - mi_bchunk_set(&bbitmap->chunkmap, chunk_idx, NULL); - mi_bbitmap_chunkmap_set_max(bbitmap, chunk_idx); -} - -static bool mi_bbitmap_chunkmap_try_clear(mi_bbitmap_t* bbitmap, size_t chunk_idx) { - mi_assert(chunk_idx < mi_bbitmap_chunk_count(bbitmap)); - // check if the corresponding chunk is all clear - if (!mi_bchunk_all_are_clear_relaxed(&bbitmap->chunks[chunk_idx])) return false; - // clear the chunkmap bit - mi_bchunk_clear(&bbitmap->chunkmap, chunk_idx, NULL); - // .. but a concurrent set may have happened in between our all-clear test and the clearing of the - // bit in the mask. We check again to catch this situation. (note: mi_bchunk_clear must be acq-rel) - if (!mi_bchunk_all_are_clear_relaxed(&bbitmap->chunks[chunk_idx])) { - mi_bchunk_set(&bbitmap->chunkmap, chunk_idx, NULL); - return false; - } - mi_bbitmap_chunkmap_set_max(bbitmap, chunk_idx); - return true; -} - - -/* -------------------------------------------------------------------------------- - mi_bbitmap_setN, try_clearN, and is_xsetN - (used to find free pages) --------------------------------------------------------------------------------- */ - -// Set a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from 0's to 1's (or 1's to 0's). -bool mi_bbitmap_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) { - mi_assert_internal(n>0); - const size_t maxbits = mi_bbitmap_max_bits(bbitmap); - mi_assert_internal(idx + n <= maxbits); - if (idx+n > maxbits) { // paranoia - if (idx >= maxbits) return false; - n = maxbits - idx; - } - - // iterate through the chunks - size_t chunk_idx = idx / MI_BCHUNK_BITS; - size_t cidx = idx % MI_BCHUNK_BITS; - bool were_allclear = true; - while (n > 0) { - const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n); - were_allclear = mi_bchunk_setN(&bbitmap->chunks[chunk_idx], cidx, m, NULL) && were_allclear; - mi_bbitmap_chunkmap_set(bbitmap, chunk_idx, true); // set afterwards - mi_assert_internal(m <= n); - n -= m; - cidx = 0; - chunk_idx++; - } - return were_allclear; -} - -// ------- mi_bbitmap_try_clearNC --------------------------------------- - -// Try to clear `n` bits at `idx` where `n <= MI_BCHUNK_BITS`. -bool mi_bbitmap_try_clearNC(mi_bbitmap_t* bbitmap, size_t idx, size_t n) { - mi_assert_internal(n>0); - mi_assert_internal(n<=MI_BCHUNK_BITS); - mi_assert_internal(idx + n <= mi_bbitmap_max_bits(bbitmap)); - - const size_t chunk_idx = idx / MI_BCHUNK_BITS; - const size_t cidx = idx % MI_BCHUNK_BITS; - mi_assert_internal(cidx + n <= MI_BCHUNK_BITS); // don't cross chunks (for now) - mi_assert_internal(chunk_idx < mi_bbitmap_chunk_count(bbitmap)); - if (cidx + n > MI_BCHUNK_BITS) return false; - bool maybe_all_clear = false; - bool did_temp_clear_bits = false; - const bool cleared = mi_bchunk_try_clearN(&bbitmap->chunks[chunk_idx], cidx, n, &maybe_all_clear, &did_temp_clear_bits); - if (cleared && maybe_all_clear) { - mi_assert_internal(!did_temp_clear_bits); - mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx); - } else if (did_temp_clear_bits) { - // may have raced with a clearer (in on_find) so set the chunkmap bit conservatively - mi_bbitmap_chunkmap_set(bbitmap, chunk_idx, false); - } - // note: we don't set the size class for an explicit try_clearN (only used by purging) - return cleared; -} - - - -// ------- mi_bbitmap_is_xset --------------------------------------- - -// Is a sequence of n bits already all set/cleared? -bool mi_bbitmap_is_xsetN(mi_xset_t set, mi_bbitmap_t* bbitmap, size_t idx, size_t n) { - mi_assert_internal(n>0); - const size_t maxbits = mi_bbitmap_max_bits(bbitmap); - mi_assert_internal(idx + n <= maxbits); - if (idx+n > maxbits) { // paranoia - if (idx >= maxbits) return false; - n = maxbits - idx; - } - - // iterate through the chunks - size_t chunk_idx = idx / MI_BCHUNK_BITS; - size_t cidx = idx % MI_BCHUNK_BITS; - bool xset = true; - while (n > 0 && xset) { - const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n); - xset = mi_bchunk_is_xsetN(set, &bbitmap->chunks[chunk_idx], cidx, m) && xset; - mi_assert_internal(m <= n); - n -= m; - cidx = 0; - chunk_idx++; - } - return xset; -} - - - - -/* -------------------------------------------------------------------------------- - mi_bbitmap_find - (used to find free pages) --------------------------------------------------------------------------------- */ - -typedef bool (mi_bchunk_try_find_and_clear_fun_t)(mi_bchunk_t* chunk, size_t n, size_t* idx, bool* did_temp_clear_bits); - -// Go through the bbitmap and for every sequence of `n` set bits, call the visitor function. -// If it returns `true` stop the search. -// -// This is used for finding free blocks and it is important to be efficient (with 2-level bitscan) -// but also reduce fragmentation (through size bins). -static bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx, mi_bchunk_try_find_and_clear_fun_t* on_find) -{ - // we space out threads to reduce contention - const size_t cmap_max_count = _mi_divide_up(mi_bbitmap_chunk_count(bbitmap),MI_BFIELD_BITS); - const size_t chunk_acc = mi_atomic_load_relaxed(&bbitmap->chunk_max_accessed); - const size_t cmap_acc = chunk_acc / MI_BFIELD_BITS; - const size_t cmap_acc_bits = 1 + (chunk_acc % MI_BFIELD_BITS); - - // create a mask over the chunkmap entries to iterate over them efficiently - mi_assert_internal(MI_BFIELD_BITS >= MI_BCHUNK_FIELDS); - const mi_bfield_t cmap_mask = mi_bfield_mask(cmap_max_count,0); - const size_t cmap_cycle = cmap_acc+1; - const mi_chunkbin_t bbin = mi_chunkbin_of(n); - // visit each cmap entry - size_t cmap_idx = 0; - mi_bfield_cycle_iterate(cmap_mask, tseq, cmap_cycle, cmap_idx, X) - { - // and for each chunkmap entry we iterate over its bits to find the chunks - const mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bbitmap->chunkmap.bfields[cmap_idx]); - const size_t cmap_entry_cycle = (cmap_idx != cmap_acc ? MI_BFIELD_BITS : cmap_acc_bits); - if (cmap_entry == 0) { - continue; - } - - // get size bin masks - mi_bfield_t cmap_bins[MI_CBIN_COUNT] = { 0 }; - cmap_bins[MI_CBIN_NONE] = cmap_entry; - for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin < MI_CBIN_NONE; ibin = mi_chunkbin_inc(ibin)) { - const mi_bfield_t cmap_bin = mi_atomic_load_relaxed(&bbitmap->chunkmap_bins[ibin].bfields[cmap_idx]); - cmap_bins[ibin] = cmap_bin & cmap_entry; - cmap_bins[MI_CBIN_NONE] &= ~cmap_bin; // clear bits that are in an assigned size bin - } - - // consider only chunks for a particular size bin at a time - // this picks the best bin only within a cmap entry (~ 1GiB address space), but avoids multiple - // iterations through all entries. - mi_assert_internal(bbin < MI_CBIN_NONE); - for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin <= MI_CBIN_NONE; - // skip from bbin to NONE (so, say, a SMALL will never be placed in a OTHER, MEDIUM, or LARGE chunk to reduce fragmentation) - ibin = (ibin == bbin ? MI_CBIN_NONE : mi_chunkbin_inc(ibin))) - { - mi_assert_internal(ibin < MI_CBIN_COUNT); - const mi_bfield_t cmap_bin = cmap_bins[ibin]; - size_t eidx = 0; - mi_bfield_cycle_iterate(cmap_bin, tseq, cmap_entry_cycle, eidx, Y) - { - // assertion doesn't quite hold as the max_accessed may be out-of-date - // mi_assert_internal(cmap_entry_cycle > eidx || ibin == MI_CBIN_NONE); - - // get the chunk - const size_t chunk_idx = cmap_idx*MI_BFIELD_BITS + eidx; - mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx]; - - size_t cidx; - bool did_temp_clear_bits = false; - if ((*on_find)(chunk, n, &cidx, &did_temp_clear_bits)) { - if (cidx==0 && ibin == MI_CBIN_NONE) { // only the first block determines the size bin - // this chunk is now reserved for the `bbin` size class - mi_bbitmap_set_chunk_bin(bbitmap, chunk_idx, bbin); - } - *pidx = (chunk_idx * MI_BCHUNK_BITS) + cidx; - mi_assert_internal(*pidx + n <= mi_bbitmap_max_bits(bbitmap)); - return true; - } - else { - // note: we may find that all are cleared only on a second iteration (when we fail to clear any bits) - // but that is ok as the chunkmap is a conservative approximation. - // todo: should _on_find_ return a boolean if there is a chance all are clear to avoid calling `try_clear?` - // probably not as we already only call `try_clear` once we fail to clear any bits. - if (did_temp_clear_bits) { - // a concurrent find_and_claim may have cleared the chunkmap bit, restore it now - mi_bbitmap_chunkmap_set(bbitmap, chunk_idx, false); - } - else { - mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx); - } - } - } - mi_bfield_cycle_iterate_end(Y); - } - } - mi_bfield_cycle_iterate_end(X); - return false; -} - -/* -------------------------------------------------------------------------------- - mi_bbitmap_try_find_and_clear -- used to find free pages - note: the compiler will fully inline the indirect function calls --------------------------------------------------------------------------------- */ - -bool mi_bbitmap_try_find_and_clear(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx) { - return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, 1, pidx, &mi_bchunk_try_find_and_clear_1); -} - -bool mi_bbitmap_try_find_and_clear8(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx) { - return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, 8, pidx, &mi_bchunk_try_find_and_clear_8); -} - -// bool mi_bbitmap_try_find_and_clearX(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx) { -// return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, MI_BFIELD_BITS, pidx, &mi_bchunk_try_find_and_clear_X); -// } - -bool mi_bbitmap_try_find_and_clearNX(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) { - mi_assert_internal(n<=MI_BFIELD_BITS); - return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, n, pidx, &mi_bchunk_try_find_and_clearNX); -} - -bool mi_bbitmap_try_find_and_clearNC(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) { - mi_assert_internal(n<=MI_BCHUNK_BITS); - return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, n, pidx, &mi_bchunk_try_find_and_clearNC); -} - - -/* -------------------------------------------------------------------------------- - mi_bbitmap_try_find_and_clear for huge objects spanning multiple chunks --------------------------------------------------------------------------------- */ - -// Try to atomically clear `n` bits starting at `chunk_idx` where `n` can span over multiple chunks -static bool mi_bchunk_try_clearN_(mi_bbitmap_t* bbitmap, size_t chunk_idx, size_t n) { - mi_assert_internal((chunk_idx * MI_BCHUNK_BITS) + n <= mi_bbitmap_max_bits(bbitmap)); - size_t m = n; // bits to go - size_t count = 0; // chunk count - while (m > 0) { - mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx + count]; - if (!mi_bchunk_try_clearN(chunk, 0, (m > MI_BCHUNK_BITS ? MI_BCHUNK_BITS : m), NULL, NULL)) { - goto rollback; - } - m = (m <= MI_BCHUNK_BITS ? 0 : m - MI_BCHUNK_BITS); - count++; - } - return true; - -rollback: - // we only need to reset chunks the we just fully cleared - while (count > 0) { - count--; - mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx + count]; - mi_bchunk_setN(chunk, 0, MI_BCHUNK_BITS, NULL); - // since we may race with clearing, we need to set the chunkmap conservatively - mi_bbitmap_chunkmap_set(bbitmap, chunk_idx + count, false); - } - return false; -} - -// Go through the bbitmap to find a sequence of `n` bits and clear them atomically where `n > MI_ARENA_MAX_CHUNK_OBJ_SIZE` -// Since these are very large object allocations we always search from the start and only consider starting at the start -// of a chunk (for fragmentation and efficiency). -// Todo: for now we try to find full empty chunks to cover `n` but we can allow a partial chunk at the end -// Todo: This scans directly through the chunks -- we might want to consult the cmap as well? -bool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) { - MI_UNUSED(tseq); - mi_assert(n > 0); if (n==0) { return false; } - - const size_t chunk_max = mi_bbitmap_chunk_count(bbitmap); - const size_t chunk_req = _mi_divide_up(n, MI_BCHUNK_BITS); // minimal number of chunks needed - if (chunk_max < chunk_req) { return false; } - - // iterate through the chunks - size_t chunk_idx = 0; - while (chunk_idx <= chunk_max - chunk_req) - { - size_t count = 0; // chunk count - do { - mi_assert_internal(chunk_idx + count < chunk_max); - mi_bchunk_t* const chunk = &bbitmap->chunks[chunk_idx + count]; - if (!mi_bchunk_all_are_set_relaxed(chunk)) { - break; - } - else { - count++; - } - } - while (count < chunk_req); - - // did we find a suitable range? - if (count == chunk_req) { - // now try to claim it! - if (mi_bchunk_try_clearN_(bbitmap, chunk_idx, n)) { - *pidx = (chunk_idx * MI_BCHUNK_BITS); - for (size_t i = 0; i < count; i++) { - mi_bbitmap_set_chunk_bin(bbitmap, chunk_idx + i, MI_CBIN_HUGE); - } - mi_assert_internal(*pidx + n <= mi_bbitmap_max_bits(bbitmap)); - return true; - } - else { - // contended: we reset count to retry from the first - // (we still skip the first chunk to guarantee progress) - count = 0; - } - } - - // keep searching but skip the scanned range - chunk_idx += count+1; - } - return false; -} - - - - - diff --git a/system/lib/mimalloc/src/arena/bitmap-chunk.h b/system/lib/mimalloc/src/arena/bitmap-chunk.h new file mode 100644 index 0000000000000..c5b3c4b206696 --- /dev/null +++ b/system/lib/mimalloc/src/arena/bitmap-chunk.h @@ -0,0 +1,93 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2026 Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#pragma once +#ifndef MI_BITMAP_CHUNK_H +#define MI_BITMAP_CHUNK_H + +// Type definitions for bitmap chunks +#include "bitmap.h" + +// Exported functions for bitmap chunks +size_t mi_bfield_popcount(mi_bfield_t x); +mi_bfield_t mi_bfield_clear_least_bit(mi_bfield_t x); +bool mi_bfield_find_least_bit(mi_bfield_t x, size_t* idx); +bool mi_bfield_find_highest_bit(mi_bfield_t x, size_t* idx); +bool mi_bfield_foreach_bit(mi_bfield_t* x, size_t* idx); +mi_bfield_t mi_bfield_mask(size_t bit_count, size_t shiftl); + +bool mi_bchunk_set(mi_bchunk_t* chunk, size_t cidx, size_t* already_set); +bool mi_bchunk_setN(mi_bchunk_t* chunk, size_t cidx, size_t n, size_t* already_set); +bool mi_bchunk_clear(mi_bchunk_t* chunk, size_t cidx, bool* all_clear); +bool mi_bchunk_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* maybe_all_clear); +size_t mi_bchunk_popcountN(mi_bchunk_t* chunk, size_t cidx, size_t n); +bool mi_bchunk_is_xsetN(mi_xset_t set, const mi_bchunk_t* chunk, size_t cidx, size_t n); + +bool mi_bchunk_try_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* maybe_all_clear, bool* did_temp_clear_bits); +bool mi_bchunk_try_find_and_clear(mi_bchunk_t* chunk, size_t* pidx); +bool mi_bchunk_try_find_and_clear_1(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits); +bool mi_bchunk_try_find_and_clear_8(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits); +mi_decl_noinline bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits); +mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits); + +void mi_bchunk_clear_once_set(mi_subproc_t* subproc, mi_bchunk_t* chunk, size_t cidx); +bool mi_bchunk_all_are_clear_relaxed(mi_bchunk_t* chunk); +bool mi_bchunk_all_are_set_relaxed(mi_bchunk_t* chunk); +bool mi_bchunk_bsr(mi_bchunk_t* chunk, size_t* pidx); +bool mi_bchunk_bsr_inv(mi_bchunk_t* chunk, size_t* pidx); +size_t mi_bchunk_popcount(mi_bchunk_t* chunk); +void mi_bchunks_unsafe_setN(mi_bchunk_t* chunks, mi_bchunkmap_t* cmap, size_t idx, size_t n); + +/* -------------------------------------------------------------------------------- + Iterate through a bfield +-------------------------------------------------------------------------------- */ + +// Cycle iteration through a bitfield. This is used to space out threads +// so there is less chance of contention. When searching for a free page we +// like to first search only the accessed part (so we reuse better). This +// high point is called the `cycle`. +// +// We then iterate through the bitfield as: +// first: [start, cycle> +// then : [0, start> +// then : [cycle, MI_BFIELD_BITS> +// +// The start is determined usually as `tseq % cycle` to have each thread +// start at a different spot. +// - We use `popcount` to improve branch prediction (maybe not needed? can we simplify?) +// - The `cycle_mask` is the part `[start, cycle>`. +#define mi_bfield_iterate(bfield,start,cycle,name_idx,SUF) { \ + mi_assert_internal(start <= cycle); \ + mi_assert_internal(start < MI_BFIELD_BITS); \ + mi_assert_internal(cycle <= MI_BFIELD_BITS); \ + const mi_bfield_t _cycle_mask##SUF = mi_bfield_mask(cycle - start, start); \ + size_t _bcount##SUF = mi_bfield_popcount(bfield); \ + mi_bfield_t _b##SUF = bfield & _cycle_mask##SUF; /* process [start, cycle> first*/\ + while(_bcount##SUF > 0) { \ + _bcount##SUF--;\ + if (_b##SUF==0) { _b##SUF = bfield & ~_cycle_mask##SUF; } /* process [0,start> + [cycle, MI_BFIELD_BITS> next */ \ + /* size_t name_idx; */ \ + const bool _found##SUF = mi_bfield_find_least_bit(_b##SUF,&name_idx); \ + _b##SUF = mi_bfield_clear_least_bit(_b##SUF); /* clear early so `continue` works */ \ + mi_assert_internal(_found##SUF); MI_UNUSED(_found##SUF); \ + { \ + +#define mi_bfield_iterate_end(SUF) \ + } \ + } \ +} + + +#define mi_bfield_cycle_iterate(bfield,tseq,cycle,name_idx,SUF) { \ + const size_t _start##SUF = (uint32_t)(tseq) % (uint32_t)(cycle); /* or: 0 to always search from the start? */\ + mi_bfield_iterate(bfield,_start##SUF,cycle,name_idx,SUF) + +#define mi_bfield_cycle_iterate_end(SUF) \ + mi_bfield_iterate_end(SUF); \ +} + +#endif diff --git a/system/lib/mimalloc/src/arena/bitmap.c b/system/lib/mimalloc/src/arena/bitmap.c new file mode 100644 index 0000000000000..d1c9228df0084 --- /dev/null +++ b/system/lib/mimalloc/src/arena/bitmap.c @@ -0,0 +1,918 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2026 Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +/* ---------------------------------------------------------------------------- +Concurrent bitmap that can set/reset sequences of bits atomically +---------------------------------------------------------------------------- */ + +#include "mimalloc.h" +#include "mimalloc/util/libc.h" +#include "mimalloc/util/stats.h" +#include "bitmap-chunk.h" +#include "bitmap.h" + +/* -------------------------------------------------------------------------------- + bitmap chunkmap +-------------------------------------------------------------------------------- */ + +static void mi_bitmap_chunkmap_set(mi_bitmap_t* bitmap, size_t chunk_idx) { + mi_assert(chunk_idx < mi_bitmap_chunk_count(bitmap)); + mi_bchunk_set(&bitmap->chunkmap, chunk_idx, NULL); +} + +static bool mi_bitmap_chunkmap_try_clear(mi_bitmap_t* bitmap, size_t chunk_idx) { + mi_assert(chunk_idx < mi_bitmap_chunk_count(bitmap)); + // check if the corresponding chunk is all clear + if (!mi_bchunk_all_are_clear_relaxed(&bitmap->chunks[chunk_idx])) return false; + // clear the chunkmap bit + mi_bchunk_clear(&bitmap->chunkmap, chunk_idx, NULL); + // .. but a concurrent set may have happened in between our all-clear test and the clearing of the + // bit in the mask. We check again to catch this situation. + if (!mi_bchunk_all_are_clear_relaxed(&bitmap->chunks[chunk_idx])) { + mi_bchunk_set(&bitmap->chunkmap, chunk_idx, NULL); + return false; + } + return true; +} + + +/* -------------------------------------------------------------------------------- + bitmap +-------------------------------------------------------------------------------- */ + +size_t mi_bitmap_size(size_t bit_count, size_t* pchunk_count) { + mi_assert_internal((bit_count % MI_BCHUNK_BITS) == 0); + bit_count = mi_align_up(bit_count, MI_BCHUNK_BITS); + mi_assert_internal(bit_count <= MI_BITMAP_MAX_BIT_COUNT); + mi_assert_internal(bit_count > 0); + const size_t chunk_count = bit_count / MI_BCHUNK_BITS; + mi_assert_internal(chunk_count >= 1); + const size_t size = offsetof(mi_bitmap_t,chunks) + (chunk_count * MI_BCHUNK_SIZE); + mi_assert_internal( (size%MI_BCHUNK_SIZE) == 0 ); + if (pchunk_count != NULL) { *pchunk_count = chunk_count; } + return size; +} + + +// initialize a bitmap to all unset; avoid a mem_zero if `already_zero` is true +// returns the size of the bitmap +size_t mi_bitmap_init(mi_bitmap_t* bitmap, size_t bit_count, bool already_zero) { + size_t chunk_count; + const size_t size = mi_bitmap_size(bit_count, &chunk_count); + if (!already_zero) { + mi_memzero_aligned(bitmap, size); + } + mi_atomic_store_release(&bitmap->chunk_count, chunk_count); + mi_assert_internal(mi_atomic_load_relaxed(&bitmap->chunk_count) <= MI_BITMAP_MAX_CHUNK_COUNT); + return size; +} + + +// Set a sequence of `n` bits in the bitmap (and can cross chunks). Not atomic so only use if local to a thread. +void mi_bitmap_unsafe_setN(mi_bitmap_t* bitmap, size_t idx, size_t n) { + mi_assert_internal(n>0); + mi_assert_internal(idx + n <= mi_bitmap_max_bits(bitmap)); + mi_bchunks_unsafe_setN(&bitmap->chunks[0], &bitmap->chunkmap, idx, n); +} + + + + +// ------- mi_bitmap_xset --------------------------------------- + +// Set a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from 0's to 1's (or 1's to 0's). +bool mi_bitmap_setN(mi_bitmap_t* bitmap, size_t idx, size_t n, size_t* palready_set) { + mi_assert_internal(n>0); + const size_t maxbits = mi_bitmap_max_bits(bitmap); + mi_assert_internal(idx + n <= maxbits); + if (idx+n > maxbits) { // paranoia + if (idx >= maxbits) return false; + n = maxbits - idx; + } + + // iterate through the chunks + size_t chunk_idx = idx / MI_BCHUNK_BITS; + size_t cidx = idx % MI_BCHUNK_BITS; + bool were_allclear = true; + size_t already_set = 0; + while (n > 0) { + const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n); + size_t _already_set = 0; + were_allclear = mi_bchunk_setN(&bitmap->chunks[chunk_idx], cidx, m, &_already_set) && were_allclear; + already_set += _already_set; + mi_bitmap_chunkmap_set(bitmap, chunk_idx); // set afterwards + mi_assert_internal(m <= n); + n -= m; + cidx = 0; + chunk_idx++; + } + if (palready_set != NULL) { *palready_set = already_set; } + return were_allclear; +} + +// Clear a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from 1's to 0's. +bool mi_bitmap_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n) { + mi_assert_internal(n>0); + const size_t maxbits = mi_bitmap_max_bits(bitmap); + mi_assert_internal(idx + n <= maxbits); + if (idx+n > maxbits) { // paranoia + if (idx >= maxbits) return false; + n = maxbits - idx; + } + + // iterate through the chunks + size_t chunk_idx = idx / MI_BCHUNK_BITS; + size_t cidx = idx % MI_BCHUNK_BITS; + bool were_allset = true; + while (n > 0) { + const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n); + bool maybe_all_clear = false; + were_allset = mi_bchunk_clearN(&bitmap->chunks[chunk_idx], cidx, m, &maybe_all_clear) && were_allset; + if (maybe_all_clear) { mi_bitmap_chunkmap_try_clear(bitmap, chunk_idx); } + mi_assert_internal(m <= n); + n -= m; + cidx = 0; + chunk_idx++; + } + return were_allset; +} + +// Count bits set in a range of `n` bits. +size_t mi_bitmap_popcountN( mi_bitmap_t* bitmap, size_t idx, size_t n) { + mi_assert_internal(n>0); + const size_t maxbits = mi_bitmap_max_bits(bitmap); + mi_assert_internal(idx + n <= maxbits); + if (idx+n > maxbits) { // paranoia + if (idx >= maxbits) return 0; + n = maxbits - idx; + } + + // iterate through the chunks + size_t chunk_idx = idx / MI_BCHUNK_BITS; + size_t cidx = idx % MI_BCHUNK_BITS; + size_t popcount = 0; + while (n > 0) { + const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n); + popcount += mi_bchunk_popcountN(&bitmap->chunks[chunk_idx], cidx, m); + mi_assert_internal(m <= n); + n -= m; + cidx = 0; + chunk_idx++; + } + return popcount; +} + + +// Set/clear a bit in the bitmap; returns `true` if atomically transitioned from 0 to 1 (or 1 to 0) +bool mi_bitmap_set(mi_bitmap_t* bitmap, size_t idx) { + return mi_bitmap_setN(bitmap, idx, 1, NULL); +} + +bool mi_bitmap_clear(mi_bitmap_t* bitmap, size_t idx) { + return mi_bitmap_clearN(bitmap, idx, 1); +} + + + +// ------- mi_bitmap_is_xset --------------------------------------- + +// Is a sequence of n bits already all set/cleared? +bool mi_bitmap_is_xsetN(mi_xset_t set, mi_bitmap_t* bitmap, size_t idx, size_t n) { + mi_assert_internal(n>0); + const size_t maxbits = mi_bitmap_max_bits(bitmap); + mi_assert_internal(idx + n <= maxbits); + if (idx+n > maxbits) { // paranoia + if (idx >= maxbits) return false; + n = maxbits - idx; + } + + // iterate through the chunks + size_t chunk_idx = idx / MI_BCHUNK_BITS; + size_t cidx = idx % MI_BCHUNK_BITS; + bool xset = true; + while (n > 0 && xset) { + const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n); + xset = mi_bchunk_is_xsetN(set, &bitmap->chunks[chunk_idx], cidx, m) && xset; + mi_assert_internal(m <= n); + n -= m; + cidx = 0; + chunk_idx++; + } + return xset; +} + +bool mi_bitmap_is_all_clear(mi_bitmap_t* bitmap) { + return mi_bitmap_is_xsetN(MI_BIT_CLEAR, bitmap, 0, mi_bitmap_max_bits(bitmap)); +} + +/* -------------------------------------------------------------------------------- + mi_bitmap_find + (used to find free pages) +-------------------------------------------------------------------------------- */ + +typedef bool (mi_bitmap_visit_fun_t)(mi_bitmap_t* bitmap, size_t chunk_idx, size_t n, size_t* idx, void* arg1, void* arg2); + +// Go through the bitmap and for every sequence of `n` set bits, call the visitor function. +// If it returns `true` stop the search. +static inline bool mi_bitmap_find(mi_bitmap_t* bitmap, size_t tseq, size_t n, size_t* pidx, mi_bitmap_visit_fun_t* on_find, void* arg1, void* arg2) +{ + const size_t chunkmap_max = mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS); + for (size_t i = 0; i < chunkmap_max; i++) { + // and for each chunkmap entry we iterate over its bits to find the chunks + const mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]); + size_t hi; + if (mi_bfield_find_highest_bit(cmap_entry, &hi)) { + size_t eidx = 0; + mi_bfield_cycle_iterate(cmap_entry, tseq%8, hi+1, eidx, Y) // reduce the tseq to 8 bins to reduce using extra memory (see `mstress`) + { + mi_assert_internal(eidx <= MI_BFIELD_BITS); + const size_t chunk_idx = i*MI_BFIELD_BITS + eidx; + mi_assert_internal(chunk_idx < mi_bitmap_chunk_count(bitmap)); + if ((*on_find)(bitmap, chunk_idx, n, pidx, arg1, arg2)) { + return true; + } + } + mi_bfield_cycle_iterate_end(Y); + } + } + return false; +} + + +/* -------------------------------------------------------------------------------- + Bitmap: try_find_and_claim -- used to allocate abandoned pages + note: the compiler will fully inline the indirect function call +-------------------------------------------------------------------------------- */ + +typedef struct mi_claim_fun_data_s { + mi_arena_t* arena; +} mi_claim_fun_data_t; + +static bool mi_bitmap_try_find_and_claim_visit(mi_bitmap_t* bitmap, size_t chunk_idx, size_t n, size_t* pidx, void* arg1, void* arg2) +{ + mi_assert_internal(n==1); MI_UNUSED(n); + mi_claim_fun_t* claim_fun = (mi_claim_fun_t*)arg1; + mi_claim_fun_data_t* claim_data = (mi_claim_fun_data_t*)arg2; + size_t cidx; + if mi_likely(mi_bchunk_try_find_and_clear(&bitmap->chunks[chunk_idx], &cidx)) { + const size_t slice_index = (chunk_idx * MI_BCHUNK_BITS) + cidx; + mi_assert_internal(slice_index < mi_bitmap_max_bits(bitmap)); + bool keep_set = true; + if ((*claim_fun)(slice_index, claim_data->arena, &keep_set)) { + // success! + mi_assert_internal(!keep_set); + *pidx = slice_index; + return true; + } + else { + // failed to claim it, set abandoned mapping again (unless the page was freed and keep_set will be false) + if (keep_set) { + const bool wasclear = mi_bchunk_set(&bitmap->chunks[chunk_idx], cidx, NULL); + mi_bitmap_chunkmap_set(bitmap, chunk_idx); + mi_assert_internal(wasclear); MI_UNUSED(wasclear); + } + } + } + else { + // we may find that all are cleared only on a second iteration but that is ok as + // the chunkmap is a conservative approximation. + mi_bitmap_chunkmap_try_clear(bitmap, chunk_idx); + } + return false; +} + +// Find a set bit in the bitmap and try to atomically clear it and claim it. +// (Used to find pages in the pages_abandoned bitmaps.) +mi_decl_nodiscard bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx, + mi_claim_fun_t* claim, mi_arena_t* arena ) +{ + mi_claim_fun_data_t claim_data = { arena }; + return mi_bitmap_find(bitmap, tseq, 1, pidx, &mi_bitmap_try_find_and_claim_visit, (void*)claim, &claim_data); +} + + +bool mi_bitmap_bsr(mi_bitmap_t* bitmap, size_t* idx) { + const size_t chunkmap_max = mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS); + for (size_t i = chunkmap_max; i > 0; ) { + i--; + mi_bfield_t cmap = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]); + size_t cmap_idx; + if (mi_bsr(cmap,&cmap_idx)) { + // from highest chunk to lowest (scan all in case the cmap entry was stale) + for (size_t j = cmap_idx+1; j>0; ) { + j--; + const size_t chunk_idx = (i*MI_BFIELD_BITS) + j; + size_t cidx; + if (mi_bchunk_bsr(&bitmap->chunks[chunk_idx], &cidx)) { + *idx = (chunk_idx * MI_BCHUNK_BITS) + cidx; + return true; + } + } + } + } + return false; +} + +// Return count of all set bits in a bitmap. +size_t mi_bitmap_popcount(mi_bitmap_t* bitmap) { + // for all chunkmap entries + size_t popcount = 0; + const size_t chunkmap_max = mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS); + for (size_t i = 0; i < chunkmap_max; i++) { + mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]); + size_t cmap_idx; + // for each chunk (corresponding to a set bit in a chunkmap entry) + while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) { + const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx; + // count bits in a chunk + popcount += mi_bchunk_popcount(&bitmap->chunks[chunk_idx]); + } + } + return popcount; +} + + + +// Clear a bit once it is set. +void mi_bitmap_clear_once_set(mi_subproc_t* subproc, mi_bitmap_t* bitmap, size_t idx) { + mi_assert_internal(idx < mi_bitmap_max_bits(bitmap)); + const size_t chunk_idx = idx / MI_BCHUNK_BITS; + const size_t cidx = idx % MI_BCHUNK_BITS; + mi_assert_internal(chunk_idx < mi_bitmap_chunk_count(bitmap)); + mi_bchunk_clear_once_set(subproc, &bitmap->chunks[chunk_idx], cidx); +} + + +// Visit all set bits in a bitmap. +// todo: optimize further? maybe use avx512 to directly get all indices using a mask_compressstore? +bool _mi_bitmap_forall_set(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg) { + // for all chunkmap entries + const size_t chunkmap_max = mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS); + for(size_t i = 0; i < chunkmap_max; i++) { + mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]); + size_t cmap_idx; + // for each chunk (corresponding to a set bit in a chunkmap entry) + while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) { + const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx; + // for each chunk field + mi_bchunk_t* const chunk = &bitmap->chunks[chunk_idx]; + for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) { + const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS); + mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[j]); + size_t bidx; + while (mi_bfield_foreach_bit(&b, &bidx)) { + const size_t idx = base_idx + bidx; + if (!visit(idx, 1, arena, arg)) return false; + } + } + } + } + return true; +} + +// Visit all set bits in a bitmap but try to return ranges (within bfields) if possible. +// Also clear those ranges atomically. +// Used by purging to purge larger ranges when possible +// todo: optimize further? maybe use avx512 to directly get all indices using a mask_compressstore? +bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg) { + // for all chunkmap entries + const size_t chunkmap_max = mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS); + for (size_t i = 0; i < chunkmap_max; i++) { + mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]); + size_t cmap_idx; + // for each chunk (corresponding to a set bit in a chunkmap entry) + while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) { + const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx; + // for each chunk field + mi_bchunk_t* const chunk = &bitmap->chunks[chunk_idx]; + for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) { + const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS); + mi_bfield_t b = mi_atomic_exchange_relaxed(&chunk->bfields[j], (mi_bfield_t)0); + #if MI_DEBUG > 1 + const size_t bpopcount = mi_popcount(b); + size_t rngcount = 0; + #endif + size_t bidx; + while (mi_bfield_find_least_bit(b, &bidx)) { + size_t rng = mi_ctz(~(b>>bidx)); // all the set bits from bidx + #if MI_DEBUG > 1 + rngcount += rng; + #endif + const size_t idx = base_idx + bidx; + mi_assert_internal(rng>=1 && rng<=MI_BFIELD_BITS); + mi_assert_internal((idx % MI_BFIELD_BITS) + rng <= MI_BFIELD_BITS); + mi_assert_internal((idx / MI_BCHUNK_BITS) < mi_bitmap_chunk_count(bitmap)); + // clear rng bits in b + b = b & ~mi_bfield_mask(rng, bidx); + if (!visit(idx, rng, arena, arg)) { + // break early: reset the non-visited bits + if (b!=0) { + mi_atomic_or_relaxed(&chunk->bfields[j], b); + } + return false; + } + } + mi_assert_internal(rngcount == bpopcount); + } + } + } + return true; +} + +// Visit all set bits in a bitmap but try to return ranges (within bfields) if possible, +// but only in chunks of at least `rngslices` slices (that are also aligned at `rngslices`) +// and clear those ranges atomically. +// However, the `rngslices` are capped at `MI_BFIELD_BITS` at most. +// Used by purging to purge larger ranges when possible. With transparent huge pages we only +// want to purge whole huge pages (2 MiB) at a time which is what the `rngslices` parameter achieves. +bool _mi_bitmap_forall_setc_rangesn(mi_bitmap_t* bitmap, size_t rngslices, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg) +{ + // use the generic routine for `rngslices<=1` (as that one finds longest ranges at a time) + if (rngslices<=1) { + return _mi_bitmap_forall_setc_ranges(bitmap, visit, arena, arg); + } + // mi_assert_internal(rngslices <= MI_BFIELD_BITS); + if (rngslices > MI_BFIELD_BITS) { rngslices = MI_BFIELD_BITS; } // cap at MI_BFIELD_BITS at most + + // for all chunkmap entries + const size_t chunkmap_max = mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS); + for (size_t i = 0; i < chunkmap_max; i++) { + mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]); + size_t cmap_idx; + // for each chunk (corresponding to a set bit in a chunkmap entry) + while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) { + const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx; + // for each chunk field + mi_bchunk_t* const chunk = &bitmap->chunks[chunk_idx]; + for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) { + const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS); + mi_bfield_t b = mi_atomic_exchange_relaxed(&chunk->bfields[j], (mi_bfield_t)0); // atomic clear + mi_bfield_t skipped = 0; // but track which bits we skip so we can restore them + size_t shift; + for(shift = 0; rngslices + shift <= MI_BFIELD_BITS; shift += rngslices) { // per `rngslices` to keep alignment + const mi_bfield_t rngmask = mi_bfield_mask(rngslices, shift); + if ((b & rngmask) == rngmask) { + const size_t idx = base_idx + shift; + if (!visit(idx, rngslices, arena, arg)) { + // break early: restore non-visited entries + mi_bfield_t notyet_visited = 0; + if (rngslices + shift < MI_BFIELD_BITS) { + notyet_visited = (b & (~(mi_bfield_t)0 << (shift + rngslices))); + } + mi_assert_internal((notyet_visited & skipped) == 0); + if ((notyet_visited | skipped) != 0) { + mi_atomic_or_relaxed(&chunk->bfields[j], notyet_visited | skipped); + } + return false; + } + } + else { + skipped = skipped | (b & rngmask); + } + } + if (shift < MI_BFIELD_BITS) { + // there are some non-visited top bits when `MI_BFIELD_BITS % rngslices != 0`. + mi_assert_internal(MI_BFIELD_BITS % rngslices != 0); + skipped = skipped | (b & (~(mi_bfield_t)0 << shift)); + } + if (skipped != 0) { + // restore non-visited entries + mi_atomic_or_relaxed(&chunk->bfields[j], skipped); + } + } + } + } + return true; +} + + +/* -------------------------------------------------------------------------------- + binned bitmap's +-------------------------------------------------------------------------------- */ + + +size_t mi_bbitmap_size(size_t bit_count, size_t* pchunk_count) { + // mi_assert_internal((bit_count % MI_BCHUNK_BITS) == 0); + bit_count = mi_align_up(bit_count, MI_BCHUNK_BITS); + mi_assert_internal(bit_count <= MI_BITMAP_MAX_BIT_COUNT); + mi_assert_internal(bit_count > 0); + const size_t chunk_count = bit_count / MI_BCHUNK_BITS; + mi_assert_internal(chunk_count >= 1); + const size_t size = offsetof(mi_bbitmap_t,chunks) + (chunk_count * MI_BCHUNK_SIZE); + mi_assert_internal( (size%MI_BCHUNK_SIZE) == 0 ); + if (pchunk_count != NULL) { *pchunk_count = chunk_count; } + return size; +} + +// initialize a bitmap to all unset; avoid a mem_zero if `already_zero` is true +// returns the size of the bitmap +size_t mi_bbitmap_init(mi_subproc_t* subproc, mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero) { + size_t chunk_count; + const size_t size = mi_bbitmap_size(bit_count, &chunk_count); + if (!already_zero) { + mi_memzero_aligned(bbitmap, size); + } + mi_atomic_store_release(&bbitmap->chunk_count, chunk_count); + mi_assert_internal(mi_atomic_load_relaxed(&bbitmap->chunk_count) <= MI_BITMAP_MAX_CHUNK_COUNT); + bbitmap->subproc = subproc; + return size; +} + +void mi_bbitmap_unsafe_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) { + mi_assert_internal(n>0); + mi_assert_internal(idx + n <= mi_bbitmap_max_bits(bbitmap)); + mi_bchunks_unsafe_setN(&bbitmap->chunks[0], &bbitmap->chunkmap, idx, n); +} + +bool mi_bbitmap_bsr_inv(mi_bbitmap_t* bbitmap, size_t* idx) { + // scan for highest zero bit in the bitmap + // note: we cannot use the chunkmap since that only conservatively denotes if there might be a set bit in a chuck + // todo: bbitmap_init rounds up the bitcount to BCHUNK_BITS and we should skip the top-padding! + const size_t chunk_count = mi_bbitmap_chunk_count(bbitmap); + for(size_t i = chunk_count; i > 0; ) { + i--; + size_t cidx; + if (mi_bchunk_bsr_inv(&bbitmap->chunks[i], &cidx)) { + *idx = (i * MI_BCHUNK_BITS) + cidx; + return true; + } + } + return false; +} + + +/* -------------------------------------------------------------------------------- + binned bitmap used to track free slices +-------------------------------------------------------------------------------- */ + +// Assign a specific size bin to a chunk +static void mi_bbitmap_set_chunk_bin(mi_bbitmap_t* bbitmap, size_t chunk_idx, mi_chunkbin_t bin) { + mi_assert_internal(chunk_idx < mi_bbitmap_chunk_count(bbitmap)); + for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin < MI_CBIN_NONE; ibin = mi_chunkbin_inc(ibin)) { + if (ibin == bin) { + const bool was_clear = mi_bchunk_set(& bbitmap->chunkmap_bins[ibin], chunk_idx, NULL); + if (was_clear) { mi_subproc_stat_increase(bbitmap->subproc, chunk_bins[ibin],1); } + } + else { + const bool was_set = mi_bchunk_clear(&bbitmap->chunkmap_bins[ibin], chunk_idx, NULL); + if (was_set) { mi_subproc_stat_decrease(bbitmap->subproc,chunk_bins[ibin],1); } + } + } +} + +mi_chunkbin_t mi_bbitmap_debug_get_bin(const mi_bchunkmap_t* chunkmap_bins, size_t chunk_idx) { + for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin < MI_CBIN_NONE; ibin = mi_chunkbin_inc(ibin)) { + if (mi_bchunk_is_xsetN(MI_BIT_SET, &chunkmap_bins[ibin], chunk_idx, 1)) { + return ibin; + } + } + return MI_CBIN_NONE; +} + +// Track the index of the highest chunk that is accessed. +static void mi_bbitmap_chunkmap_set_max(mi_bbitmap_t* bbitmap, size_t chunk_idx) { + size_t oldmax = mi_atomic_load_relaxed(&bbitmap->chunk_max_accessed); + if mi_unlikely(chunk_idx > oldmax) { + mi_atomic_cas_strong_relaxed(&bbitmap->chunk_max_accessed, &oldmax, chunk_idx); + } +} + +// Set a bit in the chunkmap +static void mi_bbitmap_chunkmap_set(mi_bbitmap_t* bbitmap, size_t chunk_idx, bool check_all_set) { + mi_assert(chunk_idx < mi_bbitmap_chunk_count(bbitmap)); + if (check_all_set) { + if (mi_bchunk_all_are_set_relaxed(&bbitmap->chunks[chunk_idx])) { + // all slices are free in this chunk: return back to the NONE bin + mi_bbitmap_set_chunk_bin(bbitmap, chunk_idx, MI_CBIN_NONE); + } + } + mi_bchunk_set(&bbitmap->chunkmap, chunk_idx, NULL); + mi_bbitmap_chunkmap_set_max(bbitmap, chunk_idx); +} + +static bool mi_bbitmap_chunkmap_try_clear(mi_bbitmap_t* bbitmap, size_t chunk_idx) { + mi_assert(chunk_idx < mi_bbitmap_chunk_count(bbitmap)); + // check if the corresponding chunk is all clear + if (!mi_bchunk_all_are_clear_relaxed(&bbitmap->chunks[chunk_idx])) return false; + // clear the chunkmap bit + mi_bchunk_clear(&bbitmap->chunkmap, chunk_idx, NULL); + // .. but a concurrent set may have happened in between our all-clear test and the clearing of the + // bit in the mask. We check again to catch this situation. (note: mi_bchunk_clear must be acq-rel) + if (!mi_bchunk_all_are_clear_relaxed(&bbitmap->chunks[chunk_idx])) { + mi_bchunk_set(&bbitmap->chunkmap, chunk_idx, NULL); + return false; + } + mi_bbitmap_chunkmap_set_max(bbitmap, chunk_idx); + return true; +} + + +/* -------------------------------------------------------------------------------- + mi_bbitmap_setN, try_clearN, and is_xsetN + (used to find free pages) +-------------------------------------------------------------------------------- */ + +// Set a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from 0's to 1's (or 1's to 0's). +bool mi_bbitmap_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) { + mi_assert_internal(n>0); + const size_t maxbits = mi_bbitmap_max_bits(bbitmap); + mi_assert_internal(idx + n <= maxbits); + if (idx+n > maxbits) { // paranoia + if (idx >= maxbits) return false; + n = maxbits - idx; + } + + // iterate through the chunks + size_t chunk_idx = idx / MI_BCHUNK_BITS; + size_t cidx = idx % MI_BCHUNK_BITS; + bool were_allclear = true; + while (n > 0) { + const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n); + were_allclear = mi_bchunk_setN(&bbitmap->chunks[chunk_idx], cidx, m, NULL) && were_allclear; + mi_bbitmap_chunkmap_set(bbitmap, chunk_idx, true); // set afterwards + mi_assert_internal(m <= n); + n -= m; + cidx = 0; + chunk_idx++; + } + return were_allclear; +} + +// ------- mi_bbitmap_try_clearNC --------------------------------------- + +// Try to clear `n` bits at `idx` where `n <= MI_BCHUNK_BITS`. +bool mi_bbitmap_try_clearNC(mi_bbitmap_t* bbitmap, size_t idx, size_t n) { + mi_assert_internal(n>0); + mi_assert_internal(n<=MI_BCHUNK_BITS); + mi_assert_internal(idx + n <= mi_bbitmap_max_bits(bbitmap)); + + const size_t chunk_idx = idx / MI_BCHUNK_BITS; + const size_t cidx = idx % MI_BCHUNK_BITS; + mi_assert_internal(cidx + n <= MI_BCHUNK_BITS); // don't cross chunks (for now) + mi_assert_internal(chunk_idx < mi_bbitmap_chunk_count(bbitmap)); + if (cidx + n > MI_BCHUNK_BITS) return false; + bool maybe_all_clear = false; + bool did_temp_clear_bits = false; + const bool cleared = mi_bchunk_try_clearN(&bbitmap->chunks[chunk_idx], cidx, n, &maybe_all_clear, &did_temp_clear_bits); + if (cleared && maybe_all_clear) { + mi_assert_internal(!did_temp_clear_bits); + mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx); + } else if (did_temp_clear_bits) { + // may have raced with a clearer (in on_find) so set the chunkmap bit conservatively + mi_bbitmap_chunkmap_set(bbitmap, chunk_idx, false); + } + // note: we don't set the size class for an explicit try_clearN (only used by purging) + return cleared; +} + + + +// ------- mi_bbitmap_is_xset --------------------------------------- + +// Is a sequence of n bits already all set/cleared? +bool mi_bbitmap_is_xsetN(mi_xset_t set, mi_bbitmap_t* bbitmap, size_t idx, size_t n) { + mi_assert_internal(n>0); + const size_t maxbits = mi_bbitmap_max_bits(bbitmap); + mi_assert_internal(idx + n <= maxbits); + if (idx+n > maxbits) { // paranoia + if (idx >= maxbits) return false; + n = maxbits - idx; + } + + // iterate through the chunks + size_t chunk_idx = idx / MI_BCHUNK_BITS; + size_t cidx = idx % MI_BCHUNK_BITS; + bool xset = true; + while (n > 0 && xset) { + const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n); + xset = mi_bchunk_is_xsetN(set, &bbitmap->chunks[chunk_idx], cidx, m) && xset; + mi_assert_internal(m <= n); + n -= m; + cidx = 0; + chunk_idx++; + } + return xset; +} + + + + +/* -------------------------------------------------------------------------------- + mi_bbitmap_find + (used to find free pages) +-------------------------------------------------------------------------------- */ + +typedef bool (mi_bchunk_try_find_and_clear_fun_t)(mi_bchunk_t* chunk, size_t n, size_t* idx, bool* did_temp_clear_bits); + +// Go through the bbitmap and for every sequence of `n` set bits, call the visitor function. +// If it returns `true` stop the search. +// +// This is used for finding free blocks and it is important to be efficient (with 2-level bitscan) +// but also reduce fragmentation (through size bins). +static bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx, mi_bchunk_try_find_and_clear_fun_t* on_find) +{ + // we space out threads to reduce contention + const size_t cmap_max_count = mi_divide_up(mi_bbitmap_chunk_count(bbitmap),MI_BFIELD_BITS); + const size_t chunk_acc = mi_atomic_load_relaxed(&bbitmap->chunk_max_accessed); + const size_t cmap_acc = chunk_acc / MI_BFIELD_BITS; + const size_t cmap_acc_bits = 1 + (chunk_acc % MI_BFIELD_BITS); + + // create a mask over the chunkmap entries to iterate over them efficiently + mi_assert_internal(MI_BFIELD_BITS >= MI_BCHUNK_FIELDS); + const mi_bfield_t cmap_mask = mi_bfield_mask(cmap_max_count,0); + const size_t cmap_cycle = cmap_acc+1; + const mi_chunkbin_t bbin = mi_chunkbin_of(n); + // visit each cmap entry + size_t cmap_idx = 0; + mi_bfield_cycle_iterate(cmap_mask, tseq, cmap_cycle, cmap_idx, X) + { + // and for each chunkmap entry we iterate over its bits to find the chunks + const mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bbitmap->chunkmap.bfields[cmap_idx]); + const size_t cmap_entry_cycle = (cmap_idx != cmap_acc ? MI_BFIELD_BITS : cmap_acc_bits); + if (cmap_entry == 0) { + continue; + } + + // get size bin masks + mi_bfield_t cmap_bins[MI_CBIN_COUNT] = { 0 }; + cmap_bins[MI_CBIN_NONE] = cmap_entry; + for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin < MI_CBIN_NONE; ibin = mi_chunkbin_inc(ibin)) { + const mi_bfield_t cmap_bin = mi_atomic_load_relaxed(&bbitmap->chunkmap_bins[ibin].bfields[cmap_idx]); + cmap_bins[ibin] = cmap_bin & cmap_entry; + cmap_bins[MI_CBIN_NONE] &= ~cmap_bin; // clear bits that are in an assigned size bin + } + + // consider only chunks for a particular size bin at a time + // this picks the best bin only within a cmap entry (~ 1GiB address space), but avoids multiple + // iterations through all entries. + mi_assert_internal(bbin < MI_CBIN_NONE); + for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin <= MI_CBIN_NONE; + // skip from bbin to NONE (so, say, a SMALL will never be placed in a OTHER, MEDIUM, or LARGE chunk to reduce fragmentation) + ibin = (ibin == bbin ? MI_CBIN_NONE : mi_chunkbin_inc(ibin))) + { + mi_assert_internal(ibin < MI_CBIN_COUNT); + const mi_bfield_t cmap_bin = cmap_bins[ibin]; + size_t eidx = 0; + mi_bfield_cycle_iterate(cmap_bin, tseq, cmap_entry_cycle, eidx, Y) + { + // assertion doesn't quite hold as the max_accessed may be out-of-date + // mi_assert_internal(cmap_entry_cycle > eidx || ibin == MI_CBIN_NONE); + + // get the chunk + const size_t chunk_idx = cmap_idx*MI_BFIELD_BITS + eidx; + mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx]; + + size_t cidx; + bool did_temp_clear_bits = false; + const bool found = (*on_find)(chunk, n, &cidx, &did_temp_clear_bits); + if (did_temp_clear_bits) { + // a concurrent find_and_claim may have cleared the chunkmap bit, restore it now + mi_bbitmap_chunkmap_set(bbitmap, chunk_idx, false); + } + if (found) { + if (cidx==0 && ibin == MI_CBIN_NONE) { // only the first block determines the size bin + // this chunk is now reserved for the `bbin` size class + mi_bbitmap_set_chunk_bin(bbitmap, chunk_idx, bbin); + } + *pidx = (chunk_idx * MI_BCHUNK_BITS) + cidx; + mi_assert_internal(*pidx + n <= mi_bbitmap_max_bits(bbitmap)); + return true; + } + else { + // note: we may find that all are cleared only on a second iteration (when we fail to clear any bits) + // but that is ok as the chunkmap is a conservative approximation. + // todo: should _on_find_ return a boolean if there is a chance all are clear to avoid calling `try_clear?` + // probably not as we already only call `try_clear` once we fail to clear any bits. + if (!did_temp_clear_bits) { + mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx); + } + } + } + mi_bfield_cycle_iterate_end(Y); + } + } + mi_bfield_cycle_iterate_end(X); + return false; +} + +/* -------------------------------------------------------------------------------- + mi_bbitmap_try_find_and_clear -- used to find free pages + note: the compiler can inline the indirect function calls in combined-source builds +-------------------------------------------------------------------------------- */ + +bool mi_bbitmap_try_find_and_clear(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx) { + return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, 1, pidx, &mi_bchunk_try_find_and_clear_1); +} + +bool mi_bbitmap_try_find_and_clear8(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx) { + return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, 8, pidx, &mi_bchunk_try_find_and_clear_8); +} + +// bool mi_bbitmap_try_find_and_clearX(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx) { +// return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, MI_BFIELD_BITS, pidx, &mi_bchunk_try_find_and_clear_X); +// } + +bool mi_bbitmap_try_find_and_clearNX(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) { + mi_assert_internal(n<=MI_BFIELD_BITS); + return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, n, pidx, &mi_bchunk_try_find_and_clearNX); +} + +bool mi_bbitmap_try_find_and_clearNC(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) { + mi_assert_internal(n<=MI_BCHUNK_BITS); + return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, n, pidx, &mi_bchunk_try_find_and_clearNC); +} + + +/* -------------------------------------------------------------------------------- + mi_bbitmap_try_find_and_clear for huge objects spanning multiple chunks +-------------------------------------------------------------------------------- */ + +// Try to atomically clear `n` bits starting at `chunk_idx` where `n` can span over multiple chunks +static bool mi_bbitmap_try_clearN_(mi_bbitmap_t* bbitmap, size_t chunk_idx, size_t n) { + mi_assert_internal((chunk_idx * MI_BCHUNK_BITS) + n <= mi_bbitmap_max_bits(bbitmap)); + size_t m = n; // bits to go + size_t count = 0; // chunk count + while (m > 0) { + mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx + count]; + bool did_temp_clear_bits = false; + if (!mi_bchunk_try_clearN(chunk, 0, (m > MI_BCHUNK_BITS ? MI_BCHUNK_BITS : m), NULL, &did_temp_clear_bits)) { + if (did_temp_clear_bits) { + // may have raced with a clearer (in on_find) so set the chunkmap bit conservatively + mi_bbitmap_chunkmap_set(bbitmap, chunk_idx + count, false); + } + goto rollback; + } + m = (m <= MI_BCHUNK_BITS ? 0 : m - MI_BCHUNK_BITS); + count++; + } + return true; + +rollback: + // we only need to reset chunks the we just fully cleared + while (count > 0) { + count--; + mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx + count]; + mi_bchunk_setN(chunk, 0, MI_BCHUNK_BITS, NULL); + // since we may race with clearing, we need to set the chunkmap conservatively + mi_bbitmap_chunkmap_set(bbitmap, chunk_idx + count, false); + } + return false; +} + +// Go through the bbitmap to find a sequence of `n` bits and clear them atomically where `n > MI_ARENA_MAX_CHUNK_OBJ_SIZE` +// Since these are very large object allocations we always search from the start and only consider starting at the start +// of a chunk (for fragmentation and efficiency). +// Todo: for now we try to find full empty chunks to cover `n` but we can allow a partial chunk at the end +// Todo: This scans directly through the chunks -- we might want to consult the cmap as well? +bool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) { + MI_UNUSED(tseq); + mi_assert(n > 0); if (n==0) { return false; } + + const size_t chunk_max = mi_bbitmap_chunk_count(bbitmap); + const size_t chunk_req = mi_divide_up(n, MI_BCHUNK_BITS); // minimal number of chunks needed + if (chunk_max < chunk_req) { return false; } + + // iterate through the chunks + size_t chunk_idx = 0; + while (chunk_idx <= chunk_max - chunk_req) + { + size_t count = 0; // chunk count + do { + mi_assert_internal(chunk_idx + count < chunk_max); + mi_bchunk_t* const chunk = &bbitmap->chunks[chunk_idx + count]; + if (!mi_bchunk_all_are_set_relaxed(chunk)) { + break; + } + else { + count++; + } + } + while (count < chunk_req); + + // did we find a suitable range? + if (count == chunk_req) { + // now try to claim it! + if (mi_bbitmap_try_clearN_(bbitmap, chunk_idx, n)) { + *pidx = (chunk_idx * MI_BCHUNK_BITS); + for (size_t i = 0; i < count; i++) { + mi_bbitmap_set_chunk_bin(bbitmap, chunk_idx + i, MI_CBIN_HUGE); + } + mi_assert_internal(*pidx + n <= mi_bbitmap_max_bits(bbitmap)); + return true; + } + else { + // contended: we reset count to retry from the first + // (we still skip the first chunk to guarantee progress) + count = 0; + } + } + + // keep searching but skip the scanned range + chunk_idx += count+1; + } + return false; +} diff --git a/system/lib/mimalloc/src/bitmap.h b/system/lib/mimalloc/src/arena/bitmap.h similarity index 99% rename from system/lib/mimalloc/src/bitmap.h rename to system/lib/mimalloc/src/arena/bitmap.h index 79eda77fad6ab..fab4ec2bccc5f 100644 --- a/system/lib/mimalloc/src/bitmap.h +++ b/system/lib/mimalloc/src/arena/bitmap.h @@ -12,6 +12,8 @@ Concurrent bitmap that can set/reset sequences of bits atomically #ifndef MI_BITMAP_H #define MI_BITMAP_H +#include "mimalloc/types.h" + /* -------------------------------------------------------------------------------- Atomic bitmaps with release/acquire guarantees: @@ -220,7 +222,7 @@ bool _mi_bitmap_forall_set(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_a // Ranges will never cross chunk boundaries though (and `slice_count <= MI_BCHUNK_BITS`) bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg); -// Visit all set bits in a bitmap with at least `rngslices` at a time (and aligned to `rngslices`). +// Visit all set bits in a bitmap with at least `rngslices` at a time (and aligned to `rngslices`). // This is used by purging to not break up transparent huge pages for example. // Ranges will never cross chunk boundaries (and `slice_count <= MI_BCHUNK_BITS`). bool _mi_bitmap_forall_setc_rangesn(mi_bitmap_t* bitmap, size_t rngslices, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg); diff --git a/system/lib/mimalloc/src/arena/readme.md b/system/lib/mimalloc/src/arena/readme.md new file mode 100644 index 0000000000000..efd6caeddf5af --- /dev/null +++ b/system/lib/mimalloc/src/arena/readme.md @@ -0,0 +1,10 @@ +# Memory Arenas and Concurrent Bitmaps + +- [`arena.c`](arena.c): Arena creation, reservation, traversal, and destruction. +- [`arena-alloc.c`](arena-alloc.c): Allocation, freeing, commitment, and purging of arena slices. +- [`arena-page.c`](arena-page.c): Allocation, abandonment, reclamation, and freeing of pages backed by arena slices. +- [`arena.h`](arena.h): Private arena declarations, traversal callbacks, and layout helpers. +- [`bitmap.c`](bitmap.c): Concurrent bitmap searches and range operations; these bitmaps are used to manage the slices in an arena. +- [`bitmap-chunk.c`](bitmap-chunk.c): Bitmaps consist of 512-bit chunks; this provides the low-level atomic bit-field and bitmap-chunk operations. +- [`bitmap.h`](bitmap.h): Bitmap types, constants, and operation declarations. +- [`bitmap-chunk.h`](bitmap-chunk.h): Private declarations and iteration helpers for bitmap fields and chunks. diff --git a/system/lib/mimalloc/src/heap.c b/system/lib/mimalloc/src/heap/heap.c similarity index 80% rename from system/lib/mimalloc/src/heap.c rename to system/lib/mimalloc/src/heap/heap.c index d7f1db1321125..e49d778455c6d 100644 --- a/system/lib/mimalloc/src/heap.c +++ b/system/lib/mimalloc/src/heap/heap.c @@ -6,9 +6,13 @@ terms of the MIT license. A copy of the license can be found in the file -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc/internal.h" -#include "mimalloc/prim.h" // _mi_prim_thread_yield -#include "mimalloc/prim-tls.h" // _mi_heap_theap +#include "mimalloc/prim/prim-tls.h" // _mi_heap_theap +#include "mimalloc/util/stats.h" +#include "mimalloc/alloc/free.h" // mi_free_subproc_safe +#include "mimalloc/alloc/page.h" +#include "mimalloc/arena/arena-page.h" +#include "mimalloc/arena/arena.h" +#include "mimalloc/heap/heap.h" /* ----------------------------------------------------------- @@ -24,6 +28,12 @@ void mi_heap_set_numa_affinity(mi_heap_t* heap, int numa_node) { heap->numa_node = (numa_node < 0 ? -1 : numa_node % _mi_os_numa_node_count()); } +void mi_heap_set_debug_fill_preserve(mi_heap_t* heap, size_t preserve) { + if (heap==NULL) { heap = mi_heap_main(); } + if (preserve > MI_MiB) { preserve = MI_MiB; } // fills are limited to 1 MiB anyway + mi_atomic_store_relaxed(&heap->debug_fill_preserve, preserve); +} + void mi_heap_stats_merge_to_subproc(mi_heap_t* heap) { if (heap==NULL) { heap = mi_heap_main(); } _mi_stats_merge_into(&heap->subproc->stats, &heap->stats); @@ -35,7 +45,7 @@ void mi_heap_stats_merge_to_main(mi_heap_t* heap) { } bool _mi_heap_theap_set(mi_heap_t* heap, mi_theap_t* theap) { - mi_assert_internal((uintptr_t)theap == 1 || _mi_theap_heap(theap)==heap); + mi_assert_internal((uintptr_t)theap == 1 || mi_theap_heap(theap)==heap); mi_assert_internal(!_mi_is_empty_theap(theap)); mi_assert_internal(heap->theap != 0); return _mi_thread_local_set(heap->theap,theap); @@ -93,20 +103,31 @@ mi_decl_cold mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap) mi_theap_t* theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); if mi_unlikely(theap==NULL) { theap = mi_heap_init_theap(heap); - if (theap==NULL) { return (mi_theap_t*)&_mi_theap_empty_wrong; } // this will return NULL from page.c:_mi_malloc_generic + if (theap==NULL) { return (mi_theap_t*)&_mi_theap_empty_wrong; } // this will return NULL from alloc/alloc-generic.c:_mi_malloc_generic } _mi_theap_cached_set(theap); return theap; } -void _mi_heap_init(mi_heap_t* heap, mi_thread_local_t theap_slot, mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id) +void _mi_heap_init(mi_heap_t* heap, mi_thread_local_t theap_slot, mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id, bool use_qsbr) { // init fields heap->theap = theap_slot; heap->subproc = subproc; heap->heap_seq = mi_atomic_increment_relaxed(&subproc->heap_total_count); heap->exclusive_arena = _mi_arena_from_id(exclusive_arena_id); + heap->reclaim_on_free = mi_option_get(mi_option_page_reclaim_on_free); heap->numa_node = -1; // no initial affinity + #if MI_QSBR + heap->qsbr_enabled = use_qsbr; + if (use_qsbr) { + if (heap->reclaim_on_free > 0) { heap->reclaim_on_free = 0; } + mi_heap_qsbr_init(&heap->qsbr); + } + #else + mi_assert_internal(!use_qsbr); + MI_UNUSED(use_qsbr); + #endif heap->profiler = mi_atomic_load_ptr_acquire(mi_profiler_t,&subproc->profiler); mi_stats_header_init(&heap->stats); mi_lock_init(&heap->theaps_lock); @@ -123,10 +144,10 @@ void _mi_heap_init(mi_heap_t* heap, mi_thread_local_t theap_slot, mi_subproc_t* } mi_atomic_increment_relaxed(&subproc->heap_count); mi_subproc_stat_increase(subproc, heaps, 1); - mi_assert_internal(_mi_is_heap_main(heap) ? heap->theap == mi_thread_local_key_fast : heap->theap != 0); + mi_assert_internal(mi_is_heap_main(heap) ? heap->theap == mi_thread_local_key_fast : heap->theap != 0); } -mi_heap_t* _mi_heap_new_for_subproc(mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id, bool is_main_heap) { +mi_heap_t* _mi_heap_new_for_subproc(mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id, bool is_main_heap, bool use_qsbr) { mi_assert_internal(is_main_heap ? (subproc->heap_main == NULL && subproc->parent != NULL) : subproc->heap_main != NULL); // heap data is allocated in the current subproc mi_heap_t* const heap_main = (is_main_heap ? subproc->parent->heap_main : subproc->heap_main); @@ -144,7 +165,7 @@ mi_heap_t* _mi_heap_new_for_subproc(mi_subproc_t* subproc, mi_arena_id_t exclusi mi_assert_internal(subproc->heap_main == NULL); subproc->heap_main = heap; } - _mi_heap_init(heap, theap_slot, subproc, exclusive_arena_id); + _mi_heap_init(heap, theap_slot, subproc, exclusive_arena_id, use_qsbr); return heap; } @@ -152,16 +173,26 @@ mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t exclusive_arena_id) { // `mi_heap_new` may be the very first mimalloc call in a process, in which case the // main heap does not exist yet and `_mi_heap_new_for_subproc` would allocate from a NULL `subproc->heap_main`. mi_thread_init(); - return _mi_heap_new_for_subproc(_mi_subproc(), exclusive_arena_id, false); + return _mi_heap_new_for_subproc(_mi_subproc(), exclusive_arena_id, false, false); } mi_heap_t* mi_heap_new(void) { return mi_heap_new_in_arena(0); } +mi_heap_t* mi_heap_new_qsbr(void) { + #if MI_QSBR + mi_thread_init(); + return _mi_heap_new_for_subproc(_mi_subproc(), 0, false, true); + #else + _mi_error_message(ENOSYS, "mi_heap_new_qsbr requires a build with QSBR support\n"); + return NULL; + #endif +} + // free all theaps belonging to this heap (without deleting their pages as we do this arena wise for efficiency) static void mi_heap_free_theaps(mi_heap_t* heap) { - // This can run concurrently with a thread that terminates (see `init.c:mi_thread_theaps_done`), + // This can run concurrently with a thread that terminates (see `heap/init.c:mi_thread_theaps_done`), // and we need to ensure we free theaps atomically. // We first detach our theaps list from any thread local lists @@ -176,16 +207,21 @@ static void mi_heap_free_theaps(mi_heap_t* heap) { theap->hnext = NULL; theap->hprev = NULL; mi_assert_internal(theap->tld==NULL); + // abandon any owned pages into the heap-wide registries so heap delete/destroy can visit them + _mi_theap_abandon_owned_pages(theap); + // clear the heap so a stale `_mi_theap_cached` entry in another thread no longer matches + // (a new heap can be allocated at the same address and would otherwise pick up this freed theap) + mi_atomic_store_ptr_release(mi_heap_t, &theap->heap, NULL); // merge stats into the owning heap stats _mi_stats_merge_into(&heap->stats, &theap->stats); // and free _mi_theap_decref(theap); // a cached entry can still point to the theap theap = next; } - } + } // set the theap thread local to NULL (so _mi_page_associated_theap does not read from a freed theap (through delete pages -> page_update_stats)) - if (!_mi_is_process_heap_main(heap)) { + if (!mi_is_process_heap_main(heap)) { _mi_thread_local_free(heap->theap); heap->theap = 0; } @@ -193,10 +229,10 @@ static void mi_heap_free_theaps(mi_heap_t* heap) { // free the heap resources (assuming the pages are already moved/destroyed, and all theaps have been freed) static void mi_heap_free(mi_heap_t* heap, bool acquire_heaps_lock) { - mi_assert_internal(heap!=NULL); // && !_mi_is_process_heap_main(heap)); + mi_assert_internal(heap!=NULL); // && !mi_is_process_heap_main(heap)); // free all arena pages infos - const bool is_main = _mi_is_heap_main(heap); + const bool is_main = mi_is_heap_main(heap); if (!is_main) { // pages for the main heap are pre-allocated in the arenas mi_lock(&heap->arena_pages_lock) { for (size_t i = 0; i < MI_MAX_ARENAS; i++) { @@ -210,8 +246,8 @@ static void mi_heap_free(mi_heap_t* heap, bool acquire_heaps_lock) { } // remove the heap from the subproc - if (!is_main) { - mi_heap_stats_merge_to_main(heap); + if (!is_main) { + mi_heap_stats_merge_to_main(heap); } else { _mi_stats_merge_into(&heap->subproc->stats,&heap->stats); @@ -227,9 +263,9 @@ static void mi_heap_free(mi_heap_t* heap, bool acquire_heaps_lock) { mi_lock_done(&heap->theaps_lock); mi_lock_done(&heap->os_abandoned_pages_lock); mi_lock_done(&heap->arena_pages_lock); - if (!_mi_is_process_heap_main(heap)) { + if (!mi_is_process_heap_main(heap)) { // _mi_thread_local_free(heap->theap); - _mi_free_subproc_safe(heap); + _mi_free_subproc_safe(heap); } } @@ -240,6 +276,10 @@ void mi_heap_delete(mi_heap_t* heap) { _mi_warning_message("cannot delete the main heap\n"); return; } + if (mi_heap_qsbr_is_enabled(heap)) { + _mi_warning_message("cannot delete a QSBR heap; use destroy after readers and mutators have stopped\n"); + return; + } mi_heap_free_theaps(heap); _mi_heap_move_pages(heap, heap_main); mi_heap_free(heap,true /* acquire subproc->heaps_lock */); @@ -252,15 +292,15 @@ void _mi_heap_force_destroy(mi_heap_t* heap, bool acquire_heaps_lock) { // if (_mi_subproc_main()->heap_main == heap) { // _mi_stats_merge_into(&heap->subproc->stats,&heap->stats); // } - // else + // else { - mi_heap_free(heap, acquire_heaps_lock); // todo: release locks of the main heap? + mi_heap_free(heap, acquire_heaps_lock); // todo: release locks of the main heap? } } void mi_heap_destroy(mi_heap_t* heap) { if (heap==NULL) return; - if (_mi_is_heap_main(heap)) { + if (mi_is_heap_main(heap)) { _mi_warning_message("cannot destroy the main heap\n"); return; } diff --git a/system/lib/mimalloc/src/init.c b/system/lib/mimalloc/src/heap/init.c similarity index 88% rename from system/lib/mimalloc/src/init.c rename to system/lib/mimalloc/src/heap/init.c index 9b0ded5b547d3..aeff139821947 100644 --- a/system/lib/mimalloc/src/init.c +++ b/system/lib/mimalloc/src/heap/init.c @@ -5,12 +5,18 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc/internal.h" #include "mimalloc/prim.h" -#include "mimalloc/prim-tls.h" +#include "mimalloc/track.h" +#include "mimalloc/prim/os.h" +#include "mimalloc/prim/prim-tls.h" +#include "mimalloc/util/random.h" +#include "mimalloc/util/stats.h" +#include "mimalloc/heap/page-map.h" +#include "mimalloc/profile/pprof.h" +#include "mimalloc/profile/sample-profile.h" -#include // memcpy, memset #include // atexit +#include "mimalloc/heap/init.h" // Empty page used to initialize the small free pages array static const mi_page_t mi_page_empty = { @@ -19,13 +25,13 @@ static const mi_page_t mi_page_empty = { #endif MI_ATOMIC_VAR_INIT(0), // xthread_id NULL, // free + NULL, // local_free + 0, // block_size {0}, // xused #if MI_SIZE_SIZE < 8 0, // xlast_used 0, // xlast_alloc #endif - NULL, // local_free - 0, // block_size 0, // page_offset 0, // capacity 0, // reserved capacity @@ -44,6 +50,11 @@ static const mi_page_t mi_page_empty = { { 0 }, // key #endif #endif + #if MI_QSBR + NULL, NULL, // qsbr_next, qsbr_prev + false, // qsbr_in_queue + mi_init_struct_zero, // qsbr + #endif }; #define MI_PAGE_EMPTY() ((mi_page_t*)&mi_page_empty) @@ -66,7 +77,7 @@ static const mi_page_t mi_page_empty = { // Empty page queues for every bin -#define QNULL(sz) { NULL, NULL, 0, (sz)*sizeof(uintptr_t) } +#define QNULL(sz) { NULL, NULL, 0, (sz)*sizeof(uintptr_t), 0 } #define MI_PAGE_QUEUES_EMPTY \ { QNULL(1), \ QNULL( 1), QNULL( 2), QNULL( 3), QNULL( 4), QNULL( 5), QNULL( 6), QNULL( 7), QNULL( 8), /* 8 */ \ @@ -115,12 +126,16 @@ static mi_decl_cache_align mi_tld_t mi_tld_detached = { NULL, // theaps list MI_LOCK_INITIALIZER, // theaps lock false, // recurse + false, // in_profiler false, // is_in_threadpool MI_MEMID_STATIC // memid }; +#ifdef __cplusplus +extern // fix warning in C++ for const variables with internal linkage +#endif mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { - MI_SMALL_PAGES_EMPTY, // direct small pages + MI_SMALL_PAGES_EMPTY, // direct small pages &mi_tld_detached, // tld MI_ATOMIC_VAR_INIT(NULL), // heap MI_ATOMIC_VAR_INIT(NULL), // subproc @@ -128,14 +143,16 @@ mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { 0, // full page retain false, // allow reclaim true, // allow abandon - true, // is_detached - ~MI_ZU(0), // sample countdown: "-1" (with a sample rate of 0, so we won't write to the empty theap with MI_SAMPLE==2) + true, // is_detached + true, // profile_never: this static empty theap must never be sampled + ~MI_ZU(0), // sample countdown: "-1" (with a sample rate of 0, so we won't write to the empty theap with MI_SAMPLE==2) 0, 0, // sample rate, requested 0, 0, // profile rate, countdown 0, 0, 0, 0, // guarded rate, countdown, min, max + 0, 0, // profiler_reserved1, profiler_reserved2 0, // heartbeat { {0}, {0}, 0, true }, // random - 0, // page count + 0, 0, // page count, OS page count MI_BIN_FULL, 0, // page retired min/max 0, // pages_full_size 0, 0, // generic count @@ -144,6 +161,9 @@ mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { MI_PAGE_QUEUES_EMPTY, MI_MEMID_STATIC, MI_STATS_NULL, // stats + #if MI_QSBR + NULL, NULL, // qsbr_first, qsbr_last + #endif }; #undef MI_STAT_COUNT @@ -185,7 +205,7 @@ static mi_tld_t* mi_tld_init(mi_tld_t* tld, size_t tseq, mi_subproc_t* subproc); // Initialize main heap static void mi_heap_main_init_once(void) { mi_memid_t memid_static = _mi_memid_create(MI_MEM_STATIC); - _mi_memcpy(&_mi_theap_empty_wrong,&_mi_theap_empty,sizeof(_mi_theap_empty_wrong)); + mi_memcpy(&_mi_theap_empty_wrong,&_mi_theap_empty,sizeof(_mi_theap_empty_wrong)); // initialize the main subprocess mi_subproc_t* subproc_main = _mi_subproc_main_init(); @@ -197,7 +217,7 @@ static void mi_heap_main_init_once(void) { // main process heap mi_process_heap_main.memid = memid_static; mi_atomic_store_ptr_release(mi_heap_t,&subproc_main->heap_main,&mi_process_heap_main); - _mi_heap_init(&mi_process_heap_main,mi_thread_local_key_fast,subproc_main,0); + _mi_heap_init(&mi_process_heap_main,mi_thread_local_key_fast,subproc_main,0,false); // detached theap for allocating meta-data (we can allocate on this without having an initialized thread) mi_process_theap_meta.memid = memid_static; @@ -206,6 +226,7 @@ static void mi_heap_main_init_once(void) { mi_process_theap_meta.page_full_retain = 2; mi_process_theap_meta.sample_rate = 0; // no sampling for meta data mi_process_theap_meta.sample_countdown = 0; + mi_theap_profile_never(&mi_process_theap_meta); // permanently exclude from profiling, see `mi_theap_profile_never` subproc_main->theap_meta = &mi_process_theap_meta; // mi_heap_theap_set(&mi_process_heap_main,&mi_process_theap_main); // set in `mi_thread_init(_theap_default)` @@ -237,6 +258,12 @@ mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc) { Thread local data ----------------------------------------------------------- */ +mi_theap_t* _mi_theap_main(void) { + // All subprocess main heaps share this slot; do not follow a possibly deleted default theap. + const mi_thread_local_t key = mi_process_heap_main.theap; + return (key==0 ? NULL : (mi_theap_t*)_mi_thread_local_get(key)); +} + static mi_tld_t* mi_tld_init(mi_tld_t* tld, size_t tseq, mi_subproc_t* subproc) { tld->subproc = subproc; tld->theaps = NULL; @@ -359,7 +386,7 @@ mi_theap_t* _mi_thread_init_with_heap(mi_heap_t* heap_main) mi_theap_t* const heap_theap = (heap_main==NULL ? NULL : (mi_theap_t*)_mi_thread_local_get(heap_main->theap)); mi_assert_internal(heap_main==NULL || heap_theap == theap); MI_UNUSED_RELEASE(heap_theap); - mi_subproc_stat_increase(_mi_theap_subproc(theap), threads, 1); // or theap stats and wait for merge? + mi_subproc_stat_increase(mi_theap_subproc(theap), threads, 1); // or theap stats and wait for merge? // _mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id()); return theap; } @@ -388,7 +415,10 @@ static void mi_thread_theaps_done(mi_tld_t* tld) mi_theap_t* next = theap->tnext; // never destroy theaps; if a dll is linked statically with mimalloc, // there may still be delete/free calls after the mi_fls_done is called. Issue #207 - _mi_theap_collect_abandon(theap); + mi_heap_t* const heap = mi_theap_heap(theap); + mi_lock(&heap->theaps_lock) { // keep page queues stable during a heap walk + _mi_theap_collect_abandon(theap); + } mi_assert_internal(theap->page_count==0); theap = next; } @@ -411,7 +441,7 @@ static void mi_thread_theaps_done(mi_tld_t* tld) while (theap != NULL) { mi_theap_t* next = theap->tnext; mi_assert_internal(theap->page_count==0); - mi_assert_internal(_mi_theap_heap_peek(theap)==NULL); + mi_assert_internal(mi_theap_heap_peek(theap)==NULL); theap->tld = NULL; theap->tnext = NULL; theap->tprev = NULL; @@ -459,7 +489,7 @@ void mi_thread_done(void) mi_attr_noexcept { } void _mi_thread_done(mi_theap_t* _theap_main) -{ +{ // NULL can be passed on some platforms if (_theap_main==NULL) { _theap_main = _mi_theap_default(); @@ -531,13 +561,16 @@ void _mi_auto_process_init(void) { mi_theap_t* theap = _mi_theap_default(); if (theap != NULL) { _mi_random_reinit_if_weak(&theap->random); - mi_subproc_t* subproc = _mi_theap_subproc(theap); + mi_subproc_t* subproc = mi_theap_subproc(theap); if (subproc->theap_meta != NULL) { mi_lock(&subproc->theap_meta_lock) { _mi_random_reinit_if_weak(&subproc->theap_meta->random); } } } + + // check if we should start the pprof profiler + _mi_pprof_profiler_init(); } @@ -562,6 +595,7 @@ static void mi_process_init_once(void) { _mi_thread_locals_init(); // pthread key create _mi_process_is_initialized = true; // before `mi_process_setup_auto_thread_done` so `_mi_theap_default` returns the current theap with `MI_TLS_RECURSE_GUARD` mi_process_setup_auto_thread_done(); // after the above mi_thread_init so it can add the current theap + _mi_fork_init(); // register fork handlers (can allocate) #if defined(_WIN32) && defined(MI_WIN_INIT_USE_FLS) // On windows, when building as a static lib the FLS cleanup happens to early for the main thread. @@ -596,15 +630,11 @@ void mi_process_init(void) mi_attr_noexcept { } } - -// Called when the process is done -static void mi_process_done_once(void) { - // only shutdown if we were initialized - if (!_mi_process_is_initialized) return; - // ensure we are called once - static bool process_done = false; - if (process_done) return; - process_done = true; +// Called once on process exit (when the process/dll is exiting gracefully) +static void mi_process_unwind_once(void) +{ + // stop the profiler + _mi_pprof_profiler_done(); // decref any cached theap _mi_theap_cached_set(_mi_theap_empty_get()); @@ -612,14 +642,14 @@ static void mi_process_done_once(void) { // release any thread specific resources and ensure _mi_thread_done is called on all but the main thread _mi_prim_thread_done_auto_done(); - #ifndef MI_SKIP_COLLECT_ON_EXIT - #if (MI_DEBUG || !defined(MI_SHARED_LIB)) - // free all memory if possible on process exit. This is not needed for a stand-alone process - // but should be done if mimalloc is statically linked into another shared library which - // is repeatedly loaded/unloaded, see issue #281. - mi_theap_collect(_mi_theap_default(), true /* force */); - #endif - #endif +#ifndef MI_SKIP_COLLECT_ON_EXIT +#if (MI_DEBUG || !defined(MI_SHARED_LIB)) + // free all memory if possible on process exit. This is not needed for a stand-alone process + // but should be done if mimalloc is statically linked into another shared library which + // is repeatedly loaded/unloaded, see issue #281. + mi_theap_collect(_mi_theap_default(), true /* force */); +#endif +#endif // done with tracking tools mi_track_done(); @@ -647,16 +677,35 @@ static void mi_process_done_once(void) { } _mi_tls_slots_done(); +} + +// Called when the process is done +static void mi_process_done_once(void) { + // only shutdown if we were initialized + if (!_mi_process_is_initialized) return; + // ensure we are called once + static bool process_done = false; + if (process_done) return; + process_done = true; + + const size_t thread_count = mi_atomic_load_acquire(&_mi_subproc_main()->thread_count); + if (thread_count == 1 || !_mi_prim_process_is_killed()) { // can we safely take locks? + mi_process_unwind_once(); + _mi_verbose_message("process done: 0x%zx\n", mi_process_tld_main.thread_id); + } + _mi_subproc_main_done(); _mi_allocator_done(); - _mi_verbose_message("process done %zu\n", sizeof(mi_page_t)); // : 0x%zx\n", mi_process_tld_main.thread_id); os_preloading = true; // don't call the C runtime anymore } // Call when the process is done (cdecl as it is used with `at_exit` on some platforms) void mi_cdecl mi_process_done(void) mi_attr_noexcept { - mi_atomic_do_once { + // we don't use `mi_atomic_do_once` here to avoid deadlocks if threads are killed due to a forced exit (issue #1377) + static _Atomic(size_t) mi_process_is_done; + size_t expected = 0; + if (mi_atomic_cas_strong_acq_rel(&mi_process_is_done, &expected, 1)) { mi_process_done_once(); } } diff --git a/system/lib/mimalloc/src/page-map.c b/system/lib/mimalloc/src/heap/page-map.c similarity index 86% rename from system/lib/mimalloc/src/page-map.c rename to system/lib/mimalloc/src/heap/page-map.c index 949e37ac3afab..74954b797fa25 100644 --- a/system/lib/mimalloc/src/page-map.c +++ b/system/lib/mimalloc/src/heap/page-map.c @@ -6,8 +6,11 @@ terms of the MIT license. A copy of the license can be found in the file -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc/internal.h" -#include "bitmap.h" +#include "mimalloc/alloc/page.h" +#include "mimalloc/arena/arena.h" +#include "mimalloc/heap/subproc.h" +#include "mimalloc/heap/page-map.h" +#include "../arena/bitmap.h" static void mi_page_map_cannot_commit(void) { _mi_warning_message("unable to commit the allocation page-map on-demand\n" ); @@ -26,7 +29,7 @@ static void mi_page_map_cannot_commit(void) { // A full 4 GiB address space (32 bit) needs only a 64 KiB page map. // Use an initial empty page map so `free(NULL)` works even if mimalloc is not yet initialized (issue #1341) -static uint8_t mi_page_map_empty[1] = { 1 }; // _mi_ptr_page(NULL) == NULL +static uint8_t mi_page_map_empty[1] = { 1 }; // mi_ptr_page(NULL) == NULL mi_decl_hidden mi_decl_cache_align _Atomic(uint8_t*) _mi_page_map = mi_page_map_empty; mi_decl_hidden _Atomic(void*) _mi_page_map_max_address = NULL; @@ -59,7 +62,7 @@ bool _mi_page_map_init(void) { mi_atomic_store_ptr_release(void, &_mi_page_map_max_address, (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits))); const size_t page_map_size = (MI_ZU(1) << (vbits - MI_ARENA_SLICE_SHIFT)); const bool commit = (page_map_size <= 1*MI_MiB || mi_option_is_enabled(mi_option_pagemap_commit)); // _mi_os_has_overcommit(); // commit on-access on Linux systems? - const size_t commit_bits = _mi_divide_up(page_map_size, MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT); + const size_t commit_bits = mi_divide_up(page_map_size, MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT); const size_t bitmap_size = (commit ? 0 : mi_bitmap_size(commit_bits, NULL)); const size_t reserve_size = bitmap_size + page_map_size; mi_subproc_t* const subproc = _mi_subproc_main(); @@ -70,7 +73,7 @@ bool _mi_page_map_init(void) { } if (mi_page_map_memid.initially_committed && !mi_page_map_memid.initially_zero) { _mi_warning_message("internal: the page map was committed but not zero initialized!\n"); - _mi_memzero_aligned(base, reserve_size); + mi_memzero_aligned(base, reserve_size); } if (bitmap_size > 0) { mi_page_map_commit = (mi_bitmap_t*)base; @@ -89,8 +92,8 @@ bool _mi_page_map_init(void) { return false; } } - mi_atomic_load_ptr_relaxed(uint8_t, &_mi_page_map)[0] = 1; // so _mi_ptr_page(NULL) == NULL - mi_assert_internal(_mi_ptr_page(NULL)==NULL); + mi_atomic_load_ptr_relaxed(uint8_t, &_mi_page_map)[0] = 1; // so mi_ptr_page(NULL) == NULL + mi_assert_internal(mi_ptr_page(NULL)==NULL); return true; } @@ -123,7 +126,7 @@ static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count) { mi_page_map_cannot_commit(); return false; } - if (!is_zero && !mi_page_map_memid.initially_zero) { _mi_memzero(start, size); } + if (!is_zero && !mi_page_map_memid.initially_zero) { mi_memzero(start, size); } mi_bitmap_set(mi_page_map_commit, i); } } @@ -140,13 +143,13 @@ static size_t mi_page_map_get_idx(mi_page_t* page, uint8_t** page_start, size_t* size_t page_size; *page_start = mi_page_area(page, &page_size); if (page_size > MI_LARGE_PAGE_SIZE) { page_size = MI_LARGE_PAGE_SIZE - MI_ARENA_SLICE_SIZE; } // furthest interior pointer - *slice_count = mi_slice_count_of_size(page_size) + ((*page_start - mi_page_slice_start(page))/MI_ARENA_SLICE_SIZE); // add for large aligned blocks - return _mi_page_map_index(page); + *slice_count = _mi_slice_count_of_size(page_size) + ((*page_start - mi_page_slice_start(page))/MI_ARENA_SLICE_SIZE); // add for large aligned blocks + return mi_page_map_index(page); } bool _mi_page_map_register(mi_page_t* page) { mi_assert_internal(page != NULL); - mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); mi_assert_internal(mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map) != NULL); // should be initialized before multi-thread access! uint8_t* page_map = mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map); if mi_unlikely(mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map) == NULL) { @@ -179,28 +182,28 @@ void _mi_page_map_unregister(mi_page_t* page) { size_t slice_count; const size_t idx = mi_page_map_get_idx(page, &page_start, &slice_count); // unset the offsets - _mi_memzero(page_map + idx, slice_count); + mi_memzero(page_map + idx, slice_count); } void _mi_page_map_unregister_range(void* start, size_t size) { uint8_t* const page_map = mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map); mi_assert_internal(page_map!=NULL); if (page_map == NULL) return; - const size_t slice_count = _mi_divide_up(size, MI_ARENA_SLICE_SIZE); - const uintptr_t index = _mi_page_map_index(start); + const size_t slice_count = mi_divide_up(size, MI_ARENA_SLICE_SIZE); + const uintptr_t index = mi_page_map_index(start); // todo: scan the commit bits and clear only those ranges? if (!mi_page_map_ensure_committed(index, slice_count)) { // we commit the range in total; return; } - _mi_memzero(&page_map[index], slice_count); + mi_memzero(&page_map[index], slice_count); } mi_page_t* _mi_safe_ptr_page(const void* p) { if mi_unlikely(p >= mi_atomic_load_ptr_relaxed(void, &_mi_page_map_max_address)) return NULL; - const uintptr_t idx = _mi_page_map_index(p); + const uintptr_t idx = mi_page_map_index(p); if mi_unlikely(mi_page_map_commit != NULL && !mi_bitmap_is_set(mi_page_map_commit, idx/MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT)) return NULL; - const uintptr_t ofs = _mi_page_map_at(idx); + const uintptr_t ofs = mi_page_map_at(idx); if mi_unlikely(ofs == 0) return NULL; return (mi_page_t*)((((uintptr_t)p >> MI_ARENA_SLICE_SHIFT) - ofs + 1) << MI_ARENA_SLICE_SHIFT); } @@ -215,12 +218,12 @@ mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_att #define MI_PAGE_MAP_SUB_SIZE (MI_PAGE_MAP_SUB_COUNT * sizeof(mi_page_t*)) // Use an initial empty page map so `free(NULL)` works even if mimalloc is not yet initialized (issue #1341) -static mi_page_map_t mi_page_map_empty = { +static mi_page_map_t mi_page_map_empty = { MI_ATOMIC_VAR_INIT(1), sizeof(mi_page_map_t), MI_MEMID_STATIC, MI_LOCK_INITIALIZER, - { MI_ATOMIC_VAR_INIT(NULL) } + { MI_ATOMIC_VAR_INIT(NULL) } }; mi_decl_hidden mi_decl_cache_align _Atomic(mi_page_map_t*) __mi_page_map = MI_ATOMIC_VAR_INIT(&mi_page_map_empty); @@ -234,12 +237,12 @@ static size_t mi_page_map_count_of_size(size_t size) { // } mi_decl_nodiscard static mi_decl_noinline bool mi_page_map_commit_entries(mi_page_map_t* pmap, size_t required_idx) { - const size_t reserved_count = mi_page_map_count_of_size(pmap->reserved_size); + const size_t reserved_count = mi_page_map_count_of_size(pmap->reserved_size); if mi_unlikely(required_idx >= reserved_count) { - mi_page_map_cannot_commit(); + mi_page_map_cannot_commit(); return false; } - size_t commit_size = _mi_align_up( sizeof(mi_page_map_t) + (required_idx * sizeof(mi_submap_t)), MI_ARENA_SLICE_SIZE ); + size_t commit_size = mi_align_up( sizeof(mi_page_map_t) + (required_idx * sizeof(mi_submap_t)), MI_ARENA_SLICE_SIZE ); if (pmap->reserved_size < commit_size) { commit_size = pmap->reserved_size; } const size_t commit_count = mi_page_map_count_of_size(commit_size); mi_assert_internal(commit_count > required_idx); @@ -251,7 +254,7 @@ mi_decl_nodiscard static mi_decl_noinline bool mi_page_map_commit_entries(mi_pag return false; } mi_assert_internal(is_zero || pmap->memid.initially_zero); - mi_atomic_store_release(&pmap->committed_count, commit_count); + mi_atomic_store_release(&pmap->committed_count, commit_count); // mi_atomic_store_release(&pmap->committed_addr, mi_page_map_addr_of_index(commit_count)); return true; } @@ -260,7 +263,7 @@ mi_decl_nodiscard static bool mi_page_map_ensure_committed(mi_page_map_t* pmap, mi_assert_internal(submap!=NULL && *submap==NULL); if mi_unlikely(idx >= mi_atomic_load_relaxed(&pmap->committed_count)) { if (idx >= mi_atomic_load_acquire(&pmap->committed_count)) { - if (!mi_page_map_commit_entries(pmap,idx)) return false; + if (!mi_page_map_commit_entries(pmap,idx)) return false; mi_assert_internal(idx < mi_atomic_load_relaxed(&pmap->committed_count)); } } @@ -268,7 +271,7 @@ mi_decl_nodiscard static bool mi_page_map_ensure_committed(mi_page_map_t* pmap, return true; } -// initialize the page map +// initialize the page map static bool mi_page_map_init_once(void) { size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_MAX_VABITS); if (vbits == 0) { @@ -292,7 +295,7 @@ static bool mi_page_map_init_once(void) { mi_assert(MI_MIN_VABITS <= vbits); const size_t reserve_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)); const size_t os_page_size = _mi_os_page_size(); - const size_t reserve_size = _mi_align_up( sizeof(mi_page_map_t) + ((reserve_count - 1) * sizeof(mi_submap_t)), os_page_size); + const size_t reserve_size = mi_align_up( sizeof(mi_page_map_t) + ((reserve_count - 1) * sizeof(mi_submap_t)), os_page_size); const size_t submap_size = MI_PAGE_MAP_SUB_SIZE; const size_t extra_reserve_size = reserve_size + submap_size; const bool commit = (vbits == MI_MIN_VABITS) || (reserve_size <= 64*MI_KiB) || // 42 virtual address bits @@ -304,21 +307,21 @@ static bool mi_page_map_init_once(void) { _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", extra_reserve_size / MI_KiB); return false; } - - // commit + + // commit size_t commit_count; if (memid.initially_committed) { if (!memid.initially_zero) { _mi_warning_message("internal: the page map was committed but not zero initialized!\n"); - _mi_memzero_aligned(pmap, extra_reserve_size); + mi_memzero_aligned(pmap, extra_reserve_size); memid.initially_zero = true; } commit_count = mi_page_map_count_of_size(reserve_size); } else { // commit first entries up to MI_MIN_VABITS entries - const size_t min_commit_count = (MI_ZU(1) << (MI_MIN_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)); - const size_t min_commit_size = _mi_align_up( sizeof(mi_page_map_t) + ((min_commit_count-1) * sizeof(mi_submap_t)), os_page_size); + const size_t min_commit_count = (MI_ZU(1) << (MI_MIN_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)); + const size_t min_commit_size = mi_align_up( sizeof(mi_page_map_t) + ((min_commit_count-1) * sizeof(mi_submap_t)), os_page_size); mi_assert_internal(min_commit_size <= reserve_size); bool is_zero; if (!_mi_os_commit(subproc,pmap,min_commit_size,&is_zero)) { @@ -330,7 +333,7 @@ static bool mi_page_map_init_once(void) { commit_count = mi_page_map_count_of_size(min_commit_size); mi_assert_internal(commit_count >= min_commit_count); } - + // ensure there is a submap for the NULL address mi_page_t** const sub0 = (mi_submap_t)((uint8_t*)pmap + reserve_size); // we reserved a submap part at the end already if (!memid.initially_committed) { @@ -341,7 +344,7 @@ static bool mi_page_map_init_once(void) { } } if (!memid.initially_zero) { // initialize low addresses with NULL - _mi_memzero_aligned(sub0, submap_size); + mi_memzero_aligned(sub0, submap_size); } // initialize the fields @@ -352,7 +355,7 @@ static bool mi_page_map_init_once(void) { // mi_atomic_store_release(&pmap->committed_addr, mi_page_map_addr_of_index(commit_count)); mi_atomic_store_ptr_release(mi_page_t*, &pmap->submaps[0], sub0); mi_atomic_store_ptr_release(mi_page_map_t, &__mi_page_map, pmap); - mi_assert_internal(_mi_ptr_page(NULL)==NULL); + mi_assert_internal(mi_ptr_page(NULL)==NULL); return true; } @@ -365,14 +368,14 @@ bool _mi_page_map_init(void) { } void _mi_page_map_unsafe_destroy(void) { - mi_page_map_t* const pmap = _mi_page_map(); + mi_page_map_t* const pmap = mi_page_map(); mi_assert_internal(pmap != NULL); if (pmap == NULL || pmap == &mi_page_map_empty) return; mi_subproc_t* const subproc = _mi_subproc_main(); - mi_lock_done(&pmap->lock); + mi_lock_done(&pmap->lock); for (size_t idx = 1; idx < pmap->committed_count; idx++) { // skip entry 0 (as we allocate that submap at the end of the page_map) - // free all sub-maps - mi_submap_t sub = _mi_page_map_at(pmap,idx); + // free all sub-maps + mi_submap_t sub = mi_page_map_at(pmap,idx); if (sub != NULL) { mi_memid_t memid = _mi_memid_create_os(sub, MI_PAGE_MAP_SUB_SIZE, true, false, false); _mi_os_free_ex(subproc, memid.mem.os.base, memid.mem.os.size, true, memid); @@ -380,23 +383,23 @@ void _mi_page_map_unsafe_destroy(void) { } } _mi_os_free_ex(subproc, pmap, pmap->reserved_size, true, pmap->memid); - mi_atomic_store_ptr_release(mi_page_map_t, &__mi_page_map, &mi_page_map_empty); + mi_atomic_store_ptr_release(mi_page_map_t, &__mi_page_map, &mi_page_map_empty); } mi_decl_nodiscard static mi_decl_noinline mi_submap_t mi_page_map_alloc_submap_at(mi_page_map_t* pmap, size_t idx) { // sub map not yet allocated, alloc now mi_submap_t sub = NULL; - mi_lock(&pmap->lock) + mi_lock(&pmap->lock) { sub = mi_atomic_load_ptr_acquire(mi_page_t*, &pmap->submaps[idx]); // reload - if (sub==NULL) // not yet allocated by another thread? + if (sub==NULL) // not yet allocated by another thread? { mi_subproc_t* const subproc = _mi_subproc_main(); mi_memid_t memid; - const size_t submap_size = MI_PAGE_MAP_SUB_SIZE; - sub = (mi_submap_t)_mi_os_zalloc(subproc, submap_size, &memid); + const size_t submap_size = MI_PAGE_MAP_SUB_SIZE; + sub = (mi_submap_t)_mi_os_zalloc(subproc, submap_size, &memid); if (sub==NULL) { - _mi_warning_message("internal error: unable to extend the page map\n"); + _mi_warning_message("internal error: unable to extend the page map\n"); } else { mi_submap_t expect = NULL; @@ -461,14 +464,14 @@ static size_t mi_page_map_get_idx(mi_page_t* page, size_t* sub_idx, size_t* slic size_t page_size; uint8_t* page_start = mi_page_area(page, &page_size); if (page_size > MI_LARGE_PAGE_SIZE) { page_size = MI_LARGE_PAGE_SIZE - MI_ARENA_SLICE_SIZE; } // furthest interior pointer - *slice_count = mi_slice_count_of_size(page_size) + ((page_start - mi_page_slice_start(page))/MI_ARENA_SLICE_SIZE); // add for large aligned blocks - return _mi_page_map_index(page_start, sub_idx); + *slice_count = _mi_slice_count_of_size(page_size) + ((page_start - mi_page_slice_start(page))/MI_ARENA_SLICE_SIZE); // add for large aligned blocks + return mi_page_map_index(page_start, sub_idx); } bool _mi_page_map_register(mi_page_t* page) { mi_assert_internal(page != NULL); - mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); - mi_page_map_t* pmap = _mi_page_map(); + mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_page_map_t* pmap = mi_page_map(); mi_assert_internal(pmap != NULL); // should be initialized before multi-thread access! if mi_unlikely(pmap == NULL) { if (!_mi_page_map_init()) return false; @@ -482,11 +485,11 @@ bool _mi_page_map_register(mi_page_t* page) { } void _mi_page_map_unregister(mi_page_t* page) { - mi_assert_internal(_mi_page_map != NULL); + mi_assert_internal(__mi_page_map != NULL); mi_assert_internal(page != NULL); - mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); - mi_page_map_t* const pmap = _mi_page_map(); - // note: should proceed even if the page was not registered yet (for failure paths in page allocation in `arena.c`) + mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_page_map_t* const pmap = mi_page_map(); + // note: should proceed even if the page was not registered yet (for failure paths in page allocation in `arena/arena-page.c`) if mi_unlikely(pmap == NULL) return; // get index and count size_t slice_count; @@ -497,17 +500,17 @@ void _mi_page_map_unregister(mi_page_t* page) { } void _mi_page_map_unregister_range(void* start, size_t size) { - mi_page_map_t* const pmap = _mi_page_map(); + mi_page_map_t* const pmap = mi_page_map(); if mi_unlikely(pmap == NULL) return; - const size_t slice_count = _mi_divide_up(size, MI_ARENA_SLICE_SIZE); + const size_t slice_count = mi_divide_up(size, MI_ARENA_SLICE_SIZE); size_t sub_idx; - const uintptr_t idx = _mi_page_map_index(start, &sub_idx); + const uintptr_t idx = mi_page_map_index(start, &sub_idx); mi_page_map_set_range(pmap, NULL, idx, sub_idx, slice_count); // todo: avoid committing if not already committed? } // Return NULL for invalid pointers mi_page_t* _mi_safe_ptr_page(const void* p) { - return _mi_checked_ptr_page(p); + return mi_checked_ptr_page(p); } mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept { diff --git a/system/lib/mimalloc/src/heap/readme.md b/system/lib/mimalloc/src/heap/readme.md new file mode 100644 index 0000000000000..66d4b2b778432 --- /dev/null +++ b/system/lib/mimalloc/src/heap/readme.md @@ -0,0 +1,9 @@ +# Heaps and Initialization + +- [`heap.c`](heap.c): First-class heap creation, destruction, ownership, and management. +- [`theap.c`](theap.c): Thread-local heap instances and page collection. +- [`init.c`](init.c): Process and thread initialization, shutdown, and initial allocator state. +- [`subproc.c`](subproc.c): Subprocess allocation domains, metadata allocation, and subprocess lifecycle management. +- [`page-map.c`](page-map.c): Address-to-page mapping to safely map any pointer +to a valid mimalloc page (or `NULL`). + diff --git a/system/lib/mimalloc/src/subproc.c b/system/lib/mimalloc/src/heap/subproc.c similarity index 74% rename from system/lib/mimalloc/src/subproc.c rename to system/lib/mimalloc/src/heap/subproc.c index f42d4a11c5208..a55b7182473bf 100644 --- a/system/lib/mimalloc/src/subproc.c +++ b/system/lib/mimalloc/src/heap/subproc.c @@ -6,8 +6,14 @@ terms of the MIT license. A copy of the license can be found in the file -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc/internal.h" -#include "mimalloc/prim-tls.h" +#include "mimalloc/prim/prim-tls.h" +#include "mimalloc/util/stats.h" +#include "mimalloc/arena/arena-alloc.h" +#include "mimalloc/arena/arena.h" +#include "mimalloc/heap/page-map.h" +#include "mimalloc/profile/sample-profile.h" +#include "mimalloc/heap/subproc.h" +#include "mimalloc/util/threadlocal.h" // pre-allocate the main subprocess structure. static mi_decl_cache_align mi_subproc_t mi_process_subproc_main = mi_init_struct_zero; @@ -19,7 +25,7 @@ static mi_lock_t mi_subprocs_lock = MI_LOCK_INITIALIZER; Meta-data allocation We allocate thread local data and theaps through a dedicated theap `subproc.theap_meta` which uses a detached tld with - a detached thread id. The initial theap_meta is statically + a detached thread id. The initial theap_meta is statically allocated and can thus be used to allocate on an as yet uninitialized thread or process. We need to take a lock though to allocate safely on the @@ -50,7 +56,7 @@ void* _mi_meta_rezalloc( mi_subproc_t* subproc, void* oldp, size_t newsize, mi_m mi_assert_internal(subproc->theap_meta != NULL); // note: since we take a meta lock we cannot use `mi_theap_rezalloc` as that could call `mi_free` which // can call `mi_stat_free` which would try to take the meta lock again. See issue #1358. - void* p = NULL; + void* p = NULL; mi_lock(&subproc->theap_meta_lock) { p = mi_theap_zalloc(subproc->theap_meta, newsize); } @@ -58,14 +64,14 @@ void* _mi_meta_rezalloc( mi_subproc_t* subproc, void* oldp, size_t newsize, mi_m if (oldp!=NULL) { const size_t oldsize = mi_usable_size(oldp); const size_t copysize = (newsize < oldsize ? newsize : oldsize); - _mi_memcpy(p,oldp,copysize); - if (memid!=NULL) { _mi_meta_free(subproc,oldp,*memid); } + mi_memcpy(p,oldp,copysize); + if (memid!=NULL) { _mi_meta_free(subproc,oldp,*memid); } else { mi_free(oldp); } } if (memid!=NULL) { *memid = _mi_memid_create_malloc(p,newsize,true); } } else { - if (memid!=NULL) { *memid = _mi_memid_none(); } + if (memid!=NULL) { *memid = _mi_memid_none(); } } return p; } @@ -76,7 +82,7 @@ void _mi_meta_free(mi_subproc_t* subproc, void* p, mi_memid_t memid) { mi_free(p); } else { - mi_assert_internal(subproc!=NULL); + mi_assert_internal(subproc!=NULL); _mi_arenas_free(subproc, p, _mi_memid_size(memid), memid); } } @@ -161,21 +167,21 @@ mi_subproc_id_t mi_subproc_new(void) { mi_memid_t memid; mi_subproc_t* const subproc = (mi_subproc_t*)_mi_meta_zalloc(parent, sizeof(mi_subproc_t), &memid); if (subproc == NULL) { return _mi_subproc_to_id(NULL); } - subproc->memid = memid; - + subproc->memid = memid; + mi_memid_t theap_memid; mi_theap_t* const theap_meta = (mi_theap_t*)_mi_meta_zalloc(parent, sizeof(mi_theap_t), &theap_memid); - if (theap_meta==NULL) { - _mi_meta_free(parent, subproc, memid); - return _mi_subproc_to_id(NULL); + if (theap_meta==NULL) { + _mi_meta_free(parent, subproc, memid); + return _mi_subproc_to_id(NULL); } theap_meta->memid = memid; - + // init subproc mi_subproc_init(subproc,parent); - + // init main heap - mi_heap_t* heap_main = _mi_heap_new_for_subproc(subproc,0,true); + mi_heap_t* heap_main = _mi_heap_new_for_subproc(subproc,0,true,false); if (heap_main==NULL) { _mi_meta_free(parent, theap_meta, theap_meta->memid); mi_subproc_destroy(_mi_subproc_to_id(subproc)); @@ -188,6 +194,7 @@ mi_subproc_id_t mi_subproc_new(void) { mi_assert_internal(parent->theap_meta->tld!=NULL); mi_assert_internal(parent->theap_meta->tld->thread_id == MI_THREADID_DETACHED); _mi_theap_init(theap_meta,heap_main,parent->theap_meta->tld /* detached tld */); + mi_theap_profile_never(theap_meta); // permanently exclude from profiling, see `mi_theap_profile_never` subproc->theap_meta = theap_meta; return _mi_subproc_to_id(subproc); @@ -222,16 +229,16 @@ static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc, bool acquire_subpro if (subproc->heap_main!=NULL) { _mi_thread_locals_thread_done(); // release thread locals that may have been allocated (safe as the main heap uses the fast key) if (_mi_subproc_is_main(subproc)) { - _mi_thread_locals_done(); + _mi_thread_locals_done(); } _mi_heap_force_destroy(subproc->heap_main, false /* don't re-acquire the heaps_lock */); // no warning if destroying the main heap } } subproc->theap_meta = NULL; // theap meta stats are merged during heap_destroy of the main heap - + if (!_mi_subproc_is_main(subproc)) { - // merge stats back into the main subproc + // merge stats back into the main subproc _mi_stats_merge_into(&mi_process_subproc_main.stats, &subproc->stats); } @@ -248,8 +255,8 @@ static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc, bool acquire_subpro // todo: should we refcount subprocesses? mi_lock_done(&subproc->arena_reserve_lock); mi_lock_done(&subproc->heaps_lock); - mi_lock_done(&subproc->theap_meta_lock); - _mi_meta_free( subproc->parent, subproc, subproc->memid); + mi_lock_done(&subproc->theap_meta_lock); + _mi_meta_free( subproc->parent, subproc, subproc->memid); if (_mi_subproc_is_main(subproc)) { // for the main subproc, also release the global page map _mi_page_map_unsafe_destroy(); @@ -325,3 +332,65 @@ void _mi_subproc_main_done(void) { mi_lock_done(&mi_subprocs_lock); } + +/* ----------------------------------------------------------- + Fork support + + `fork()` is only safe if no other thread is executing inside mimalloc + at the time of the fork; all other threads must be quiesced at a safe + point outside mimalloc (as is done for example by CPython's stop-the-world). + The child then inherits consistent data structures with no locks held + (as no thread was inside mimalloc) so we only need to abandon the pages + of the threads that do not exist in the child. +----------------------------------------------------------- */ + +// Abandon the pages of theaps whose threads do not exist in the child. +// The theap and tld memory itself is not freed as the (dead) thread lists may be inconsistent. +// We need to take no locks as at this point only the child process runs in a single thread. +static void mi_fork_child_abandon_theaps(void) { + mi_theap_t* const theap_default = _mi_theap_default(); + mi_tld_t* const tld_current = (mi_theap_is_initialized(theap_default) ? theap_default->tld : NULL); + mi_theap_t* dead = NULL; + for (mi_subproc_t* subproc = mi_subprocs; subproc != NULL; subproc = subproc->next) { + for (mi_heap_t* heap = subproc->heaps; heap != NULL; heap = heap->next) { + mi_theap_t* theap = heap->theaps; + while (theap != NULL) { + mi_theap_t* const next = theap->hnext; + mi_tld_t* const tld = theap->tld; + if (tld != NULL && tld != tld_current && tld->thread_id != MI_THREADID_DETACHED) { + // remove from the heap list and push on the dead list + if (theap->hnext != NULL) { theap->hnext->hprev = theap->hprev; } + if (theap->hprev != NULL) { theap->hprev->hnext = theap->hnext; } + else { heap->theaps = theap->hnext; } + theap->hprev = NULL; + theap->hnext = dead; + dead = theap; + } + theap = next; + } + } + } + while (dead != NULL) { + mi_theap_t* const next = dead->hnext; + mi_heap_t* const heap = mi_theap_heap(dead); + dead->hnext = NULL; + dead->tld = NULL; + _mi_theap_abandon_owned_pages(dead); + _mi_stats_merge_into(&heap->stats, &dead->stats); + mi_atomic_store_ptr_release(mi_heap_t, &dead->heap, NULL); + dead = next; + } +} + +void _mi_fork_child(void) { + mi_fork_child_abandon_theaps(); +} + +// Called at process initialization. +void _mi_fork_init(void) { + #if defined(MI_USE_PTHREADS) && !defined(__wasi__) && !defined(__EMSCRIPTEN__) && !MI_NO_PTHREAD_ATFORK + if (pthread_atfork(NULL, NULL, &_mi_fork_child) != 0) { + _mi_warning_message("unable to register the pthread fork handler\n"); + } + #endif +} \ No newline at end of file diff --git a/system/lib/mimalloc/src/theap.c b/system/lib/mimalloc/src/heap/theap.c similarity index 74% rename from system/lib/mimalloc/src/theap.c rename to system/lib/mimalloc/src/heap/theap.c index 1731830a7aa41..8266bd01cf7ba 100644 --- a/system/lib/mimalloc/src/theap.c +++ b/system/lib/mimalloc/src/heap/theap.c @@ -6,9 +6,15 @@ terms of the MIT license. A copy of the license can be found in the file -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc/internal.h" -#include "mimalloc/prim.h" // _mi_prim_thread_yield -#include "mimalloc/prim-tls.h" // _mi_theap_default +#include "mimalloc/prim.h" // _mi_prim_thread_yield +#include "mimalloc/prim/prim-tls.h" // _mi_theap_default +#include "mimalloc/util/random.h" +#include "mimalloc/util/stats.h" +#include "mimalloc/alloc/page.h" +#include "mimalloc/arena/arena-alloc.h" +#include "mimalloc/profile/sample-guarded.h" +#include "mimalloc/heap/heap.h" +#include "mimalloc/heap/theap.h" #if defined(_MSC_VER) && (_MSC_VER < 1920) #pragma warning(disable:4204) // non-constant aggregate initializer @@ -58,17 +64,18 @@ static bool mi_theap_page_is_valid(mi_theap_t* theap, mi_page_queue_t* pq, mi_pa MI_UNUSED(pq); mi_assert_internal(mi_page_theap(page) == theap); mi_theap_t* const page_theap = _mi_heap_theap_peek(page->heap); - mi_assert_internal(page_theap == NULL || theap == page_theap); + mi_assert_internal(page_theap == NULL || theap == page_theap || mi_theap_is_detached(theap)); mi_assert_expensive(_mi_page_is_valid(page)); return true; } static bool mi_theap_is_valid(mi_theap_t* theap) { mi_assert_internal(theap!=NULL); - mi_heap_t* const heap = _mi_theap_heap_peek(theap); + mi_heap_t* const heap = mi_theap_heap_peek(theap); mi_assert_internal(heap != NULL); mi_theap_t* const heap_theap = _mi_heap_theap_peek(heap); // don't use mi_heap_theap as that may re-initialize the thread - mi_assert_internal(heap_theap==NULL || heap_theap == theap); + // Detached metadata theaps are shared and need not match the calling thread's TLS. + mi_assert_internal(heap_theap==NULL || heap_theap == theap || mi_theap_is_detached(theap)); mi_theap_visit_pages(theap, &mi_theap_page_is_valid, true, NULL, NULL); for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) { mi_assert_internal(_mi_page_queue_is_valid(theap, &theap->pages[bin])); @@ -100,24 +107,23 @@ static bool mi_theap_page_collect(mi_theap_t* theap, mi_page_queue_t* pq, mi_pag mi_assert_expensive(mi_theap_page_is_valid(theap, pq, page, NULL, NULL)); mi_collect_t collect = *((mi_collect_t*)arg_collect); _mi_page_free_collect(page, collect >= MI_FORCE); // update used count - _mi_page_update_stats(page); + _mi_page_update_stats(page); if (mi_page_all_free(page)) { // no more used blocks, possibly free the page. if (collect >= MI_FORCE || page->retire_expire == 0) { // either forced/abandon, or not already retired // note: this will potentially free retired pages as well. - _mi_page_free(page, pq); + if (_mi_page_free(page, pq)) return true; } } - else if (collect == MI_ABANDON) { - // still used blocks but the thread is done; abandon the page + if (collect == MI_ABANDON) { _mi_page_abandon(page, pq); } return true; // don't break } void _mi_theap_merge_stats(mi_theap_t* theap) { - mi_assert_internal(mi_theap_is_initialized(theap)); - mi_heap_t* const heap = _mi_theap_heap(theap); + mi_assert_internal(mi_theap_is_initialized(theap)); + mi_heap_t* const heap = mi_theap_heap(theap); _mi_stats_merge_into(&heap->stats, &theap->stats); } @@ -127,13 +133,15 @@ static void mi_theap_collect_ex(mi_theap_t* theap, mi_collect_t collect) mi_assert_expensive(mi_theap_is_valid(theap)); const bool force = (collect >= MI_FORCE); - _mi_deferred_free(theap, force); + if (collect != MI_ABANDON) { // skip on thread shutdown to avoid running user-code after process exit (issue #1377) + _mi_deferred_free(theap, force); + } // python/cpython#112532: we may be called from a thread that is not the owner of the theap // const bool is_main_thread = (_mi_is_main_thread() && theap->thread_id == _mi_thread_id()); // collect retired pages (and full pages if theap->allow_page_abandon is false) - _mi_theap_collect_retired(theap, force); + _mi_theap_collect_retired(theap, force); // collect all pages owned by this thread mi_theap_visit_pages(theap, &mi_theap_page_collect, (collect!=MI_NORMAL), &collect, NULL); // dont normally visit full pages, see issue #1220 @@ -145,7 +153,7 @@ static void mi_theap_collect_ex(mi_theap_t* theap, mi_collect_t collect) } // merge statistics - if (mi_option_is_enabled(mi_option_collect_merges_stats)) { + if (mi_option_is_enabled(mi_option_stats_merge_on_collect)) { _mi_theap_merge_stats(theap); } } @@ -154,6 +162,25 @@ void _mi_theap_collect_abandon(mi_theap_t* theap) { mi_theap_collect_ex(theap, MI_ABANDON); } +// Abandon all owned pages of a valid but detached theap (`tld==NULL`) into the heap-wide registries. +void _mi_theap_abandon_owned_pages(mi_theap_t* theap) { + mi_assert_internal(theap!=NULL); + mi_assert_internal(theap->tld == NULL); // should be detached as we may access from another thread + if (theap==NULL) return; + for (size_t bin = 0; bin <= MI_BIN_FULL; bin++) { + mi_page_queue_t* const pq = &theap->pages[bin]; + mi_page_t* page = pq->first; + while (page != NULL) { + mi_page_t* const next = page->next; // save as the page is removed from the queue + _mi_page_free_collect(page, true); // collect cross-thread frees so `used` is precise + if (!mi_page_all_free(page) || !_mi_page_free(page, pq)) { + _mi_page_abandon(page, pq); + } + page = next; + } + } +} + void mi_theap_collect(mi_theap_t* theap, bool force) mi_attr_noexcept { mi_theap_collect_ex(theap, (force ? MI_FORCE : MI_NORMAL)); } @@ -191,9 +218,9 @@ mi_theap_t* mi_theap_set_default(mi_theap_t* theap) { } -static void mi_theap_options_init(mi_theap_t* theap) { - theap->allow_page_reclaim = (mi_option_get(mi_option_page_reclaim_on_free) >= 0); - theap->allow_page_abandon = (mi_option_get(mi_option_page_full_retain) >= 0); +static void mi_theap_options_init(mi_theap_t* theap, mi_heap_t* heap) { + theap->allow_page_reclaim = (mi_heap_reclaim_on_free(heap) >= 0); + theap->allow_page_abandon = (mi_option_get(mi_option_page_full_retain) >= 0); // nowadays qsbr pages can be abandoned: && !mi_heap_qsbr_is_enabled(heap)); theap->page_full_retain = mi_option_get_clamp(mi_option_page_full_retain, -1, 32); theap->is_detached = (theap->tld->thread_id == MI_THREADID_DETACHED); } @@ -205,17 +232,18 @@ void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld) mi_assert_internal(heap!=NULL); mi_assert_internal(tld!=NULL); mi_memid_t memid = theap->memid; - _mi_memcpy_aligned(theap, &_mi_theap_empty, sizeof(mi_theap_t)); + mi_memcpy_aligned(theap, &_mi_theap_empty, sizeof(mi_theap_t)); theap->memid = memid; theap->tld = tld; // avoid reading the thread-local tld during initialization - mi_atomic_store_release(&theap->refcount,1); + theap->profile_never = false; // only set for the empty and meta theaps (see `mi_theap_profile_never`) + mi_atomic_store_release(&theap->refcount,1); mi_atomic_store_ptr_release(mi_subproc_t,&theap->subproc,heap->subproc); mi_assert_internal(theap->stats.size == sizeof(mi_stats_t)); - mi_theap_options_init(theap); + mi_theap_options_init(theap, heap); if (theap->tld->is_in_threadpool) { // if we run as part of a thread pool it is better to not arbitrarily reclaim abandoned pages into our theap. - // this is checked in `free.c:mi_free_try_collect_mt` + // this is checked in `alloc/free.c:mi_free_try_collect_mt` // .. but abandoning is good in this case: quarter the full page retain (possibly to 0) // (so blocked threads do not hold on to too much memory) if (theap->page_full_retain > 0) { @@ -231,23 +259,26 @@ void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld) theap->tprev = NULL; theap->tnext = head; theap->tld->theaps = theap; - if (head!=NULL) { - head->tprev = theap; + if (head!=NULL) { + head->tprev = theap; head_random = head->random; - } + } } // initialize random if heap==NULL if (head==NULL) { // first theap of the first thread? - #if defined(_WIN32) && !defined(MI_SHARED_LIB) - if (tld->thread_seq==0) { - _mi_random_init_weak(&theap->random); // prevent allocation failure during bcrypt dll initialization with static linking (issue #1185) - } - else - #endif - { - _mi_random_init(&theap->random); - } + // will be initialized on the first random_next/split call. (see #1377, W20) + + // #if defined(_WIN32) && !defined(MI_SHARED_LIB) + // if (tld->thread_seq==0) { + // _mi_random_init_weak(&theap->random); // prevent allocation failure during bcrypt dll initialization with static linking (issue #1185) + // } + // else + // #endif + // { + // _mi_random_init(&theap->random); + // } + } else { _mi_random_split(&head_random, &theap->random); // &theap->random is used as nonce so it is ok if threads capture the same head->random @@ -255,12 +286,12 @@ void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld) // theap->cookie = _mi_theap_random_next(theap) | 1; _mi_theap_guarded_init(theap); // needs theap->random if (!theap->is_detached) { - mi_subproc_stat_increase(_mi_theap_subproc(theap),theaps,1); // on subproc to match theap_free_mem + mi_subproc_stat_increase(mi_theap_subproc(theap),theaps,1); // on subproc to match theap_free_mem } // only now set the heap member as it is used to determine if a theap is initialized mi_atomic_store_ptr_release(mi_heap_t,&theap->heap,heap); - + // push on the heap's theap list mi_lock(&heap->theaps_lock) { head = heap->theaps; @@ -280,15 +311,15 @@ mi_theap_t* _mi_theap_alloc(mi_heap_t* heap, mi_tld_t* tld) { // allocate and initialize a theap mi_memid_t memid; mi_theap_t* theap; - + if (heap->exclusive_arena == NULL) { theap = (mi_theap_t*)_mi_meta_zalloc(heap->subproc, sizeof(mi_theap_t), &memid); } else { // theaps associated with a specific arena are allocated in that arena // note: takes up at least one slice which is quite wasteful... - const size_t size = _mi_align_up(sizeof(mi_theap_t),MI_ARENA_MIN_OBJ_SIZE); - theap = (mi_theap_t*)_mi_arenas_alloc(heap, size, true, true, heap->exclusive_arena, tld->thread_seq, tld->numa_node, &memid); + const size_t size = mi_align_up(sizeof(mi_theap_t),MI_ARENA_MIN_OBJ_SIZE); + theap = (mi_theap_t*)_mi_arenas_alloc(heap, size, true, true, heap->exclusive_arena, tld->thread_seq, tld->numa_node, &memid); } if (theap==NULL) { _mi_error_message(ENOMEM, "unable to allocate theap meta-data\n"); @@ -306,15 +337,18 @@ mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld) { return theap; } -uintptr_t _mi_theap_random_next(mi_theap_t* theap) { +size_t _mi_theap_random_next(mi_theap_t* theap) { return _mi_random_next(&theap->random); } static void mi_theap_free_mem(mi_theap_t* theap) { if (theap!=NULL) { - mi_subproc_t* const subproc = mi_atomic_load_ptr_relaxed(mi_subproc_t,&theap->subproc); + #if MI_QSBR + mi_assert_internal(theap->qsbr_first == NULL && theap->qsbr_last == NULL); + #endif + mi_subproc_t* const subproc = mi_atomic_load_ptr_relaxed(mi_subproc_t,&theap->subproc); if (!theap->is_detached) { - mi_subproc_stat_decrease(subproc,theaps,1); + mi_subproc_stat_decrease(subproc,theaps,1); } _mi_meta_free(subproc, theap, theap->memid); } @@ -335,7 +369,7 @@ void _mi_theap_decref(mi_theap_t* theap) { } } -// Thread termination and heap delete/destroy might run concurrently +// Thread termination and heap delete/destroy might run concurrently // and we need to ensure we free the memory correctly. A heap or tld // will first "detach" its theaps so it has a list with theaps that are // no longer shared, and only then free's the theaps in that list. @@ -343,6 +377,25 @@ void _mi_theap_decref(mi_theap_t* theap) { // Due to lock-inversion we need to use `mi_lock_try_acquire` and if that fails // we back-off, release the outer lock, and try again until we succeed. +// Both the heap and tld list locks must be held. +static void mi_theap_detach_from_tld(mi_theap_t* theap) { + mi_tld_t* const tld = theap->tld; + if (theap->tnext != NULL) { theap->tnext->tprev = theap->tprev; } + if (theap->tprev != NULL) { theap->tprev->tnext = theap->tnext; } + else { mi_assert_internal(tld->theaps == theap); tld->theaps = theap->tnext; } + theap->tnext = theap->tprev = NULL; + theap->tld = NULL; +} + +static void mi_theap_detach_from_heap(mi_theap_t* theap) { + mi_heap_t* const heap = mi_theap_heap(theap); + if (theap->hnext != NULL) { theap->hnext->hprev = theap->hprev; } + if (theap->hprev != NULL) { theap->hprev->hnext = theap->hnext; } + else { mi_assert_internal(heap->theaps == theap); heap->theaps = theap->hnext; } + theap->hnext = theap->hprev = NULL; + mi_atomic_store_ptr_release(mi_heap_t, &theap->heap, NULL); +} + // Remove the theaps in this heap from any thread local tld lists. void _mi_heap_detach_theaps( mi_heap_t* heap ) { bool all_detached; @@ -352,15 +405,10 @@ void _mi_heap_detach_theaps( mi_heap_t* heap ) { mi_theap_t* theap = heap->theaps; while (theap != NULL) { mi_theap_t* next = theap->hnext; - mi_tld_t* tld = theap->tld; + mi_tld_t* tld = theap->tld; if (tld != NULL) { if (mi_lock_try_acquire(&tld->theaps_lock)) { - // remove the theap from the tld theaps list - if (theap->tnext != NULL) { theap->tnext->tprev = theap->tprev; } - if (theap->tprev != NULL) { theap->tprev->tnext = theap->tnext; } - else { mi_assert_internal(theap->tld->theaps == theap); theap->tld->theaps = theap->tnext; } - theap->tnext = theap->tprev = NULL; - theap->tld = NULL; + mi_theap_detach_from_tld(theap); mi_lock_release(&tld->theaps_lock); } else { @@ -387,18 +435,12 @@ void _mi_tld_detach_theaps( mi_tld_t* tld ) { while (theap != NULL) { mi_theap_t* next = theap->tnext; mi_assert_internal(theap->page_count==0); - mi_heap_t* heap = _mi_theap_heap_peek(theap); // now the heap might be NULL from an earlier iteration + mi_heap_t* heap = mi_theap_heap_peek(theap); // now the heap might be NULL from an earlier iteration if (heap != NULL) { if (mi_lock_try_acquire(&heap->theaps_lock)) { // merge stats into the owning heap stats _mi_stats_merge_into(&heap->stats, &theap->stats); - // remove the theap from the heap list - if (theap->hnext != NULL) { theap->hnext->hprev = theap->hprev; } - if (theap->hprev != NULL) { theap->hprev->hnext = theap->hnext; } - else { mi_assert_internal(heap->theaps == theap); heap->theaps = theap->hnext; } - theap->hnext = theap->hprev = NULL; - // and set `heap` to NULL - mi_atomic_store_ptr_release(mi_heap_t, &theap->heap, NULL); + mi_theap_detach_from_heap(theap); mi_lock_release(&heap->theaps_lock); } else { @@ -421,22 +463,68 @@ void _mi_tld_detach_theaps( mi_tld_t* tld ) { Safe theap delete ----------------------------------------------------------- */ -// Safe delete a theap without freeing any still allocated blocks in that theap. -// void _mi_theap_delete(mi_theap_t* theap, bool acquire_tld_theaps_lock) -// { -// mi_assert(theap != NULL); -// mi_assert(mi_theap_is_initialized(theap)); -// mi_assert_expensive(mi_theap_is_valid(theap)); -// if (theap==NULL || !mi_theap_is_initialized(theap)) return; - -// // abandon all pages -// _mi_theap_collect_abandon(theap); - -// mi_assert_internal(theap->page_count==0); -// _mi_theap_free(theap, true /* acquire heap->theaps_lock */, acquire_tld_theaps_lock); -// } +void mi_theap_delete(mi_theap_t* theap) mi_attr_noexcept { + if (theap==NULL) return; + // The main theap stays alive even if a concurrent heap delete invalidates our default. + mi_theap_t* const main = _mi_theap_main(); + if (main==NULL) { + _mi_warning_message("cannot delete a theap on an uninitialized thread\n"); + return; + } + mi_tld_t* const tld = main->tld; + if (_mi_theap_default() == theap) { + _mi_theap_default_set(main); + } + + bool found = false; + bool release = false; + mi_lock(&tld->theaps_lock) { + // Do not dereference the argument until list membership protects it from heap deletion. + mi_theap_t* owned = tld->theaps; + while (owned!=NULL && owned!=theap) { owned = owned->tnext; } + if (owned!=NULL) { + found = true; + mi_heap_t* const heap = mi_theap_heap(owned); + mi_assert_internal(heap!=NULL); + mi_lock(&heap->theaps_lock) { + if (owned == main) { + _mi_theap_collect_abandon(owned); // retain the backing theap and tld for the living thread + _mi_theap_merge_stats(owned); + } + else { + const bool cleared = _mi_thread_local_set(heap->theap, NULL); + mi_assert_internal(cleared); MI_UNUSED_RELEASE(cleared); + mi_theap_detach_from_tld(owned); + _mi_theap_abandon_owned_pages(owned); + _mi_theap_merge_stats(owned); + mi_theap_detach_from_heap(owned); + release = true; + } + } + } + } + if (_mi_theap_cached() == theap) { + _mi_theap_cached_set(_mi_theap_empty_get()); + } + if (release) { _mi_theap_decref(theap); } + if (!found) { _mi_verbose_message("theap %p is no longer owned by this thread\n", theap); } +} +void mi_theap_quiescent_abandon(mi_theap_t* theap) mi_attr_noexcept { + if (theap==NULL || !mi_theap_is_initialized(theap)) return; + mi_tld_t* const tld = theap->tld; + mi_heap_t* const heap = mi_theap_heap(theap); + mi_lock_maybe(&tld->theaps_lock, tld!=NULL) { + mi_lock(&heap->theaps_lock) { + // Temporarily detach for foreign-thread page cleanup, without changing any TLS references. + theap->tld = NULL; + _mi_theap_abandon_owned_pages(theap); + theap->tld = tld; + _mi_theap_merge_stats(theap); + } + } +} /* ----------------------------------------------------------- Load/unload theaps @@ -446,7 +534,7 @@ void mi_theap_unload(mi_theap_t* theap) { mi_assert(mi_theap_is_initialized(theap)); mi_assert_expensive(mi_theap_is_valid(theap)); if (theap==NULL || !mi_theap_is_initialized(theap)) return; - if (_mi_theap_heap(theap)->exclusive_arena == NULL) { + if (mi_theap_heap(theap)->exclusive_arena == NULL) { _mi_warning_message("cannot unload theaps that are not associated with an exclusive arena\n"); return; } @@ -466,7 +554,7 @@ void mi_theap_unload(mi_theap_t* theap) { bool mi_theap_reload(mi_theap_t* theap, mi_arena_id_t arena_id) { mi_assert(mi_theap_is_initialized(theap)); if (theap==NULL || !mi_theap_is_initialized(theap)) return false; - if (_mi_theap_heap(theap)->exclusive_arena == NULL) { + if (mi_theap_heap(theap)->exclusive_arena == NULL) { _mi_warning_message("cannot reload theaps that were not associated with an exclusive arena\n"); return false; } @@ -475,8 +563,8 @@ bool mi_theap_reload(mi_theap_t* theap, mi_arena_id_t arena_id) { return false; } mi_arena_t* arena = _mi_arena_from_id(arena_id); - if (_mi_theap_heap(theap)->exclusive_arena != arena) { - _mi_warning_message("trying to reload a theap at a different arena address: %p vs %p\n", _mi_theap_heap(theap)->exclusive_arena, arena); + if (mi_theap_heap(theap)->exclusive_arena != arena) { + _mi_warning_message("trying to reload a theap at a different arena address: %p vs %p\n", mi_theap_heap(theap)->exclusive_arena, arena); return false; } @@ -565,7 +653,7 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi // create a bitmap of free blocks. #define MI_MAX_BLOCKS (MI_SMALL_PAGE_SIZE / sizeof(void*)) uintptr_t free_map[MI_MAX_BLOCKS / MI_INTPTR_BITS]; - const uintptr_t bmapsize = _mi_divide_up(page->capacity, MI_INTPTR_BITS); + const uintptr_t bmapsize = mi_divide_up(page->capacity, MI_INTPTR_BITS); memset(free_map, 0, bmapsize * sizeof(intptr_t)); if (page->capacity % MI_INTPTR_BITS != 0) { // mark left-over bits at the end as free @@ -654,6 +742,7 @@ static bool mi_theap_visit_areas_page(mi_theap_t* theap, mi_page_queue_t* pq, mi mi_theap_area_visit_fun* fun = (mi_theap_area_visit_fun*)vfun; mi_theap_area_ex_t xarea; xarea.page = page; + _mi_page_free_collect(page, true); // collect cross-thread free'd blocks so `area.used` is precise _mi_heap_area_init(&xarea.area, page); return fun(theap, &xarea, arg); } @@ -673,7 +762,7 @@ typedef struct mi_visit_blocks_args_s { static bool mi_theap_area_visitor(const mi_theap_t* theap, const mi_theap_area_ex_t* xarea, void* arg) { mi_visit_blocks_args_t* args = (mi_visit_blocks_args_t*)arg; - if (!args->visitor(_mi_theap_heap(theap), &xarea->area, NULL, xarea->area.block_size, args->arg)) return false; + if (!args->visitor(mi_theap_heap(theap), &xarea->area, NULL, xarea->area.block_size, args->arg)) return false; if (args->visit_blocks) { return _mi_theap_area_visit_blocks(&xarea->area, xarea->page, args->visitor, args->arg); } @@ -687,4 +776,3 @@ bool mi_theap_visit_blocks(const mi_theap_t* theap, bool visit_blocks, mi_block_ mi_visit_blocks_args_t args = { visit_blocks, visitor, arg }; return mi_theap_visit_areas(theap, &mi_theap_area_visitor, &args); } - diff --git a/system/lib/mimalloc/src/prim/emscripten/prim.c b/system/lib/mimalloc/src/prim/emscripten/prim.c index 4386456baa7e3..1a605a294e85a 100644 --- a/system/lib/mimalloc/src/prim/emscripten/prim.c +++ b/system/lib/mimalloc/src/prim/emscripten/prim.c @@ -8,9 +8,8 @@ terms of the MIT license. A copy of the license can be found in the file // This file is included in `src/prim/prim.c` #include "mimalloc.h" -#include "mimalloc/internal.h" -#include "mimalloc/atomic.h" #include "mimalloc/prim.h" +#include "mimalloc/heap/init.h" #include // sched_yield #include // getentropy @@ -151,7 +150,7 @@ size_t _mi_prim_numa_node_count(void) { mi_msecs_t _mi_prim_clock_now(void) { // todo: use a monotonic clock instead - return emscripten_date_now(); + return emscripten_date_now(); } @@ -208,7 +207,8 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { // use pthread local storage keys to detect thread ending // (and used with MI_TLS_PTHREADS for the default theap) -pthread_key_t _mi_heap_default_key = MI_PTHREAD_KEY_INVALID; +// atomic as a thread may initialize while the process is exiting and deleting the key +static _Atomic(pthread_key_t) _mi_heap_default_key = MI_ATOMIC_VAR_INIT(MI_PTHREAD_KEY_INVALID); static void mi_pthread_done(void* value) { if (value!=NULL) { @@ -217,21 +217,24 @@ static void mi_pthread_done(void* value) { } void _mi_prim_thread_init_auto_done(void) { - mi_assert_internal(_mi_heap_default_key == MI_PTHREAD_KEY_INVALID); - pthread_key_create(&_mi_heap_default_key, &mi_pthread_done); + mi_assert_internal(mi_atomic_load_relaxed(&_mi_heap_default_key) == MI_PTHREAD_KEY_INVALID); + pthread_key_t key; + if (pthread_key_create(&key, &mi_pthread_done) == 0) { + mi_atomic_store_release(&_mi_heap_default_key, key); + } } void _mi_prim_thread_done_auto_done(void) { - pthread_key_t key = _mi_heap_default_key; + pthread_key_t key = mi_atomic_exchange_relaxed(&_mi_heap_default_key, MI_PTHREAD_KEY_INVALID); if (key != MI_PTHREAD_KEY_INVALID) { // do not leak the key, see issue #809 - _mi_heap_default_key = MI_PTHREAD_KEY_INVALID; pthread_key_delete(key); } } void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) { - if (_mi_heap_default_key != MI_PTHREAD_KEY_INVALID) { // can happen during recursive invocation on freeBSD - pthread_setspecific(_mi_heap_default_key, theap); + pthread_key_t key = mi_atomic_load_relaxed(&_mi_heap_default_key); + if (key != MI_PTHREAD_KEY_INVALID) { // can happen during recursive invocation on freeBSD + pthread_setspecific(key, theap); } } @@ -257,3 +260,7 @@ bool _mi_prim_thread_is_in_threadpool(void) { void _mi_prim_thread_yield(void) { sched_yield(); } + +bool _mi_prim_process_is_killed(void) { + return false; +} diff --git a/system/lib/mimalloc/src/os.c b/system/lib/mimalloc/src/prim/os.c similarity index 95% rename from system/lib/mimalloc/src/os.c rename to system/lib/mimalloc/src/prim/os.c index 15111cf94988c..c9debd50f1035 100644 --- a/system/lib/mimalloc/src/os.c +++ b/system/lib/mimalloc/src/prim/os.c @@ -5,10 +5,13 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc/internal.h" -#include "mimalloc/atomic.h" #include "mimalloc/prim.h" -#include "mimalloc/prim-tls.h" // _mi_theap_default for random +#include "mimalloc/track.h" +#include "mimalloc/prim/prim-tls.h" // _mi_theap_default for random +#include "mimalloc/util/libc.h" +#include "mimalloc/util/options.h" +#include "mimalloc/util/stats.h" +#include "mimalloc/prim/os.h" /* ----------------------------------------------------------- Initialization. @@ -41,7 +44,7 @@ bool _mi_os_has_virtual_reserve(void) { return mi_os_mem_config.has_virtual_reserve; } -bool _mi_os_canuse_thp(void) { +bool _mi_os_has_thp(void) { return mi_os_mem_config.has_transparent_huge_pages; } @@ -58,14 +61,14 @@ size_t _mi_os_large_page_size(void) { // minimal purge size. Can be larger than the page size if transparent huge pages are enabled. size_t _mi_os_minimal_purge_size(void) { size_t minsize = mi_option_get_size(mi_option_minimal_purge_size); - if (minsize != 0) { + if (minsize != 0) { // set by user - return _mi_align_up(minsize, _mi_os_page_size()); + return mi_align_up(minsize, _mi_os_page_size()); } else if (mi_os_mem_config.has_transparent_huge_pages && mi_option_get(mi_option_allow_thp) == 2) { - // don't break up THP pages; + // don't break up THP pages; // we don't do this by default as that can lead to increased memory usage (see issue #1282). - // on the other hand, breaking up THP pages can lead to performance degradation and perhaps + // on the other hand, breaking up THP pages can lead to performance degradation and perhaps // allow_thp should be 2 by default (and the user can set the OS THP setting to [never]). return _mi_os_large_page_size(); } @@ -102,7 +105,7 @@ size_t _mi_os_good_alloc_size(size_t size) { else if (size < 32*MI_MiB) align_size = 1*MI_MiB; else align_size = 4*MI_MiB; if mi_unlikely(size >= (SIZE_MAX - align_size)) return size; // possible overflow? - return _mi_align_up(size, align_size); + return mi_align_up(size, align_size); } void _mi_os_init(void) { @@ -139,14 +142,14 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t sze) // todo: perhaps only do alignment hints if THP is enabled? if (try_alignment <= mi_os_mem_config.alloc_granularity || try_alignment > 16*MI_GiB) return NULL; if (mi_os_mem_config.virtual_address_bits < 46) return NULL; // < 64TiB virtual address space - + size_t req_size = sze + _mi_os_page_size(); // always reserve a bit more to create virtual gaps between hinted blocks. req_size += (try_alignment - 1); // ensure we can align in the requested size - req_size = _mi_align_up(req_size, _mi_os_large_page_size()); + req_size = mi_align_up(req_size, _mi_os_large_page_size()); #if (MI_SECURE>=1) if (req_size > 32*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<34) = 1/256 #endif - + uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, req_size); if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize uintptr_t init = MI_HINT_BASE; @@ -161,7 +164,7 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t sze) hint = mi_atomic_add_acq_rel(&aligned_base, req_size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all if (hint==0) return NULL; } - const uintptr_t hint_align = _mi_align_up(hint,try_alignment); + const uintptr_t hint_align = mi_align_up(hint,try_alignment); mi_assert_internal(hint_align + sze < hint + req_size); return (void*)hint_align; } @@ -319,7 +322,7 @@ static void* mi_os_prim_alloc_at(mi_subproc_t* subproc, void* hint_addr, size_t // try to align along large OS page size for larger allocations const size_t large_page_size = mi_os_mem_config.large_page_size; - if (large_page_size > 0 && hint_addr == NULL && size >= 8*large_page_size && _mi_is_power_of_two(try_alignment) && try_alignment < large_page_size) { + if (large_page_size > 0 && hint_addr == NULL && size >= 8*large_page_size && mi_is_power_of_two(try_alignment) && try_alignment < large_page_size) { try_alignment = large_page_size; } @@ -359,7 +362,7 @@ static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t *memid = _mi_memid_none(); if (!commit) allow_large = false; if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL; - size = _mi_align_up(size, _mi_os_page_size()); + size = mi_align_up(size, _mi_os_page_size()); #if MI_SIZE_SIZE >= 8 const bool try_direct_alloc = true; @@ -378,12 +381,14 @@ static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t } // aligned already? - if (p != NULL && _mi_is_aligned(p,alignment)) { + if (p != NULL && mi_is_aligned(p,alignment)) { os_base = p; } else { // if not aligned, free it, overallocate, and unmap around it - #if !MI_TRACK_ASAN + // (define MI_NO_ALIGN_FALLBACK_WARNING to silence this benign warning, e.g. when + // stderr output is checked by tests or the address space layout is unusual) + #if !MI_TRACK_ASAN && !defined(MI_NO_ALIGN_FALLBACK_WARNING) if (try_direct_alloc) { _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit); } @@ -402,7 +407,7 @@ static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t // this is handled though by having the `base` field in the memid os_base = p; // remember the base os_size = over_size; // todo: use size instead as now we over-decrement commit stats on free? - p = _mi_align_up_ptr(p, alignment); + p = mi_align_up_ptr(p, alignment); // explicitly commit only the aligned part if (commit) { @@ -419,9 +424,9 @@ static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t if (p == NULL) return NULL; // and selectively unmap parts around the over-allocated area. - void* const aligned_p = _mi_align_up_ptr(p, alignment); + void* const aligned_p = mi_align_up_ptr(p, alignment); const size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p; - const size_t mid_size = _mi_align_up(size, _mi_os_page_size()); + const size_t mid_size = mi_align_up(size, _mi_os_page_size()); const size_t post_size = over_size - pre_size - mid_size; mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size); if (pre_size > 0) { mi_os_prim_free(subproc, p, pre_size, (commit ? pre_size : 0), true /* adjust */); } @@ -433,7 +438,7 @@ static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t } } - mi_assert_internal(p != NULL && os_base != NULL && _mi_is_aligned(p,alignment)); + mi_assert_internal(p != NULL && os_base != NULL && mi_is_aligned(p,alignment)); mi_assert_internal(os_base <= p && size <= os_size); *memid = _mi_memid_create_os(os_base,os_size,commit,os_is_zero,os_is_large); return p; @@ -465,13 +470,13 @@ void* _mi_os_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, *memid = _mi_memid_none(); if (size == 0) return NULL; size = _mi_os_good_alloc_size(size); - alignment = _mi_align_up(alignment, _mi_os_page_size()); + alignment = mi_align_up(alignment, _mi_os_page_size()); void* p = mi_os_prim_alloc_aligned(subproc, size, alignment, commit, allow_large, memid ); if (p == NULL) return NULL; mi_assert_internal(memid->mem.os.size >= size); - mi_assert_internal(_mi_is_aligned(p,alignment)); + mi_assert_internal(mi_is_aligned(p,alignment)); if (commit) { mi_assert_internal(memid->initially_committed); } return p; } @@ -490,7 +495,7 @@ mi_decl_nodiscard static void* mi_os_ensure_zero(mi_subproc_t* subproc, void* p, } // ensure zero'd if (memid->initially_zero) return p; - _mi_memzero_aligned(p,size); + mi_memzero_aligned(p,size); memid->initially_zero = true; return p; } @@ -519,14 +524,14 @@ void* _mi_os_alloc_aligned_at_offset(mi_subproc_t* subproc, size_t size, size_t } else { // overallocate to align at an offset - const size_t extra = _mi_align_up(offset, alignment) - offset; + const size_t extra = mi_align_up(offset, alignment) - offset; if (size >= SIZE_MAX - extra) return NULL; // too large const size_t oversize = size + extra; void* const start = _mi_os_alloc_aligned(subproc, oversize, alignment, commit, allow_large, memid); if (start == NULL) return NULL; void* const p = (uint8_t*)start + extra; - mi_assert(_mi_is_aligned((uint8_t*)p + offset, alignment)); + mi_assert(mi_is_aligned((uint8_t*)p + offset, alignment)); // decommit the overallocation at the start // note: this double counts the decommit when freeing `memid`. Should we keep commit size in the memid as well? if (commit && extra >= _mi_os_page_size()) { @@ -548,10 +553,10 @@ static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size, if (size == 0 || addr == NULL) return NULL; // page align conservatively within the range, or liberally straddling pages outside the range - void* start = (conservative ? _mi_align_up_ptr(addr, _mi_os_page_size()) - : _mi_align_down_ptr(addr, _mi_os_page_size())); - void* end = (conservative ? _mi_align_down_ptr((uint8_t*)addr + size, _mi_os_page_size()) - : _mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size())); + void* start = (conservative ? mi_align_up_ptr(addr, _mi_os_page_size()) + : mi_align_down_ptr(addr, _mi_os_page_size())); + void* end = (conservative ? mi_align_down_ptr((uint8_t*)addr + size, _mi_os_page_size()) + : mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size())); ptrdiff_t diff = (uint8_t*)end - (uint8_t*)start; if (diff <= 0) return NULL; @@ -649,7 +654,7 @@ bool _mi_os_reset(mi_subproc_t* subproc, void* addr, size_t size) { } -void _mi_os_reuse( mi_subproc_t* subproc, void* addr, size_t size ) { +void _mi_os_reuse( mi_subproc_t* subproc, void* addr, size_t size ) { MI_UNUSED(subproc); // page align conservatively within the range size_t csize = 0; @@ -924,7 +929,7 @@ static void* mi_os_alloc_aligned_ex(size_t size, size_t alignment, bool commit, if (base != NULL) { *base = memid.mem.os.base; } if (full_size != NULL) { *full_size = memid.mem.os.size; } if (!memid.initially_zero && memid.initially_committed) { - _mi_memzero_aligned(memid.mem.os.base, memid.mem.os.size); + mi_memzero_aligned(memid.mem.os.base, memid.mem.os.size); } return p; } diff --git a/system/lib/mimalloc/src/prim/osx/alloc-override-zone.c b/system/lib/mimalloc/src/prim/osx/alloc-override-zone.c index 48f97958218d9..00ba0fa88ad26 100644 --- a/system/lib/mimalloc/src/prim/osx/alloc-override-zone.c +++ b/system/lib/mimalloc/src/prim/osx/alloc-override-zone.c @@ -6,8 +6,7 @@ terms of the MIT license. A copy of the license can be found in the file -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc/internal.h" -#include "mimalloc/prim-tls.h" // _mi_thread_is_initialized +#include "mimalloc/prim/os.h" #if defined(MI_MALLOC_OVERRIDE) #if !defined(__APPLE__) @@ -25,7 +24,6 @@ terms of the MIT license. A copy of the license can be found in the file #include #include -#include // memset #include #ifdef __cplusplus @@ -41,10 +39,10 @@ extern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_im malloc zone members ------------------------------------------------------ */ -static bool is_mimalloc_zone( malloc_zone_t* zone ); +static bool is_mimalloc_zone( malloc_zone_t* zone ); static size_t zone_size(malloc_zone_t* zone, const void* p) { - if (mi_any_heap_contains(p)) { + if (mi_any_heap_contains(p)) { return mi_usable_size(p); } else if (!is_mimalloc_zone(zone)) { // can happen due to interpose @@ -71,7 +69,7 @@ static void* zone_valloc(malloc_zone_t* zone, size_t size) { } static void zone_free(malloc_zone_t* zone, void* p) { - // during C++ thread shutdown `_pthread_tsd_cleanup` may call `zone_free` + // during C++ thread shutdown `_pthread_tsd_cleanup` may call `zone_free` // after mimalloc mi_thread_done, and also on a pointer that was allocated in another subproc. if mi_unlikely(!mi_cfree(p)) { if (!is_mimalloc_zone(zone)) { // can happen due to interpose @@ -226,7 +224,7 @@ static malloc_introspection_t mi_introspect = { #if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__) .statistics = &intro_statistics, .zone_locked = &intro_zone_locked, -#endif +#endif #if defined(MAC_OS_X_VERSION_10_12) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_12) && !defined(__ppc__) .reinit_lock = &intro_reinit_lock, #endif @@ -282,7 +280,7 @@ static bool is_mimalloc_zone( malloc_zone_t* zone ) { // our mimalloc zone. Since even the loader uses malloc // on macOS, this ensures that all allocations go through // mimalloc (as all calls are interposed). -// The main `malloc`, `free`, etc calls are interposed in `alloc-override.c`, +// The main `malloc`, `free`, etc calls are interposed in `alloc/alloc-override.c`, // Here, we also override macOS specific API's like // `malloc_zone_calloc` etc. see // ------------------------------------------------------ @@ -384,16 +382,9 @@ static void zone_unregister(malloc_zone_t* zone) { // use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1` // See: -struct mi_interpose_s { - const void* replacement; - const void* target; -}; -#define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun } -#define MI_INTERPOSE_MI(fun) MI_INTERPOSE_FUN(fun,mi_##fun) #define MI_INTERPOSE_ZONE(fun) MI_INTERPOSE_FUN(malloc_##fun,fun) -__attribute__((used)) static const struct mi_interpose_s _mi_zone_interposes[] __attribute__((section("__DATA, __interpose"))) = +MI_INTERPOSE_DECLS(_mi_zone_interposes) = { - MI_INTERPOSE_MI(malloc_create_zone), MI_INTERPOSE_MI(malloc_default_purgeable_zone), MI_INTERPOSE_MI(malloc_default_zone), diff --git a/system/lib/mimalloc/src/prim/prim-tls.c b/system/lib/mimalloc/src/prim/prim-tls.c index 1b79321a05ef0..ddaa5d4576fb2 100644 --- a/system/lib/mimalloc/src/prim/prim-tls.c +++ b/system/lib/mimalloc/src/prim/prim-tls.c @@ -6,13 +6,13 @@ terms of the MIT license. A copy of the license can be found in the file -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc/internal.h" #include "mimalloc/prim.h" -#include "mimalloc/prim-tls.h" +#include "mimalloc/prim/prim-tls.h" +#include "mimalloc/util/options.h" // _mi_error_message // -------------------------------------------------------------------------- // Implement fast access to the thread local storage for `_mi_theap_default()` -// and `mi_theap_cached()`. See `include/mimalloc/prim-tls.h` for more info +// and `mi_theap_cached()`. See `include/mimalloc/prim/prim-tls.h` for more info // on the TLS models. // -------------------------------------------------------------------------- @@ -55,7 +55,7 @@ mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { #if !MI_WIN_DIRECT_TLS // We initially use the last of the expansion slots as the default NULL. -// note: this will fail if the program allocates exactly 1024+64 slots with TlsAlloc +// note: this will fail if the program allocates exactly 1024+64 slots with TlsAlloc // before we are initialized :-( (but this seems quite unlikely). // (todo: another approach could be to use slot 7 (EnvironmentPointer) as the initial slot as that seems to be always NULL) #define MI_TLS_INITIAL_SLOT MI_TLS_EXPANSION_SLOT @@ -166,7 +166,7 @@ void _mi_tls_slots_init(void) { pthread_key_t key; if (_mi_pthread_key_create(&key,NULL,NULL)) { mi_atomic_store_release(&_mi_theap_default_key,key); } if (_mi_pthread_key_create(&key,&mi_theap_cached_key_destroy,NULL)) { mi_atomic_store_release(&_mi_theap_cached_key,key); } - } + } } void _mi_tls_slots_done(void) { @@ -176,7 +176,7 @@ void _mi_tls_slots_done(void) { if (key!=MI_PTHREAD_KEY_INVALID) { pthread_key_delete(key); } } -#elif MI_TLS_MODEL_FIXED +#elif MI_TLS_MODEL_FIXED void _mi_tls_slots_init(void) { mi_atomic_do_once { diff --git a/system/lib/mimalloc/src/prim/prim.c b/system/lib/mimalloc/src/prim/prim.c index 3d3d0c2e8c881..17ab7ad8bfe59 100644 --- a/system/lib/mimalloc/src/prim/prim.c +++ b/system/lib/mimalloc/src/prim/prim.c @@ -5,6 +5,10 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ +#include "mimalloc/heap/init.h" +#include "mimalloc/prim.h" + + // Select the implementation of the primitives // depending on the OS. @@ -14,13 +18,13 @@ terms of the MIT license. A copy of the license can be found in the file #elif defined(__APPLE__) #include "osx/prim.c" // macOSX (actually defers to mmap in unix/prim.c) +#elif defined(__EMSCRIPTEN__) +#include "emscripten/prim.c" // emmalloc_*, + pthread support + #elif defined(__wasi__) #define MI_USE_SBRK #include "wasi/prim.c" // memory-grow or sbrk (Wasm) -#elif defined(__EMSCRIPTEN__) -#include "emscripten/prim.c" // emmalloc_*, + pthread support - #else #include "unix/prim.c" // mmap() (Linux, macOSX, BSD, Illumnos, Haiku, DragonFly, etc.) diff --git a/system/lib/mimalloc/src/prim/readme.md b/system/lib/mimalloc/src/prim/readme.md index 02cb1b90c8c0e..1917c1a3d4e4e 100644 --- a/system/lib/mimalloc/src/prim/readme.md +++ b/system/lib/mimalloc/src/prim/readme.md @@ -1,12 +1,8 @@ -## Portability Primitives +# OS and Portability Primitives -This is the portability layer where all primitives needed from the OS are defined. +- [`include/mimalloc/prim.h`](/src/include/mimalloc/prim.h): primitive portability API definition. +- [`prim.c`](prim.c): This is the portability layer where all primitives needed from the OS are defined. + Selects one of [`unix/prim.c`](unix/prim.c), [`windows/prim.c`](windows/prim.c), [`wasi/prim.c`](wasi/prim.c), or [`emscripten/prim.c`](emscripten/prim.c) depending on the host platform (and on macOS, [`osx/prim.c`](osx/prim.c) defers to [`unix/prim.c`](unix/prim.c)). +- [`prim-tls.c`](prim-tls.c): primitive TLS model (thread local, pthreads, etc.) +- [`os.c`](os.c): A general higher level API around [`prim.c`](prim.c) that supports OS allocation, (de)commit, huge page allocation, etc. -- `include/mimalloc/prim.h`: primitive portability API definition. -- `prim.c`: Selects one of `unix/prim.c`, `wasi/prim.c`, or `windows/prim.c` depending on the host platform - (and on macOS, `osx/prim.c` defers to `unix/prim.c`). - -- `include/mimalloc/prim-tls.h`: primitive TLS model (thread local, pthreads, etc) -- `prim-tls.c`: TLS model implementation - -Note: still work in progress, there may still be places in the sources that still depend on OS ifdef's. \ No newline at end of file diff --git a/system/lib/mimalloc/src/prim/unix/prim.c b/system/lib/mimalloc/src/prim/unix/prim.c index 25c2adea039f8..794d2d070f43f 100644 --- a/system/lib/mimalloc/src/prim/unix/prim.c +++ b/system/lib/mimalloc/src/prim/unix/prim.c @@ -21,8 +21,12 @@ terms of the MIT license. A copy of the license can be found in the file #endif #include "mimalloc.h" -#include "mimalloc/internal.h" #include "mimalloc/prim.h" +#include "mimalloc/track.h" +#include "mimalloc/prim/os.h" +#include "mimalloc/util/libc.h" +#include "mimalloc/util/options.h" +#include "mimalloc/heap/init.h" #include // mmap #include // sysconf, sleep @@ -38,8 +42,6 @@ terms of the MIT license. A copy of the license can be found in the file #endif #if defined(__GLIBC__) #include // linux mmap flags - #else - #include #endif #if defined(__riscv) || defined(_M_RISCV) #if defined(MI_HAS_SYS_HWPROBEH) @@ -72,6 +74,17 @@ terms of the MIT license. A copy of the license can be found in the file #include #endif +// MSan does not intercept raw syscalls, so memory they write must be marked as initialized +#if defined(__has_feature) + #if __has_feature(memory_sanitizer) + #include // __msan_unpoison + #define mi_msan_unpoison(p,size) __msan_unpoison(p,size) + #endif +#endif +#ifndef mi_msan_unpoison +#define mi_msan_unpoison(p,size) +#endif + #if !defined(MADV_DONTNEED) && defined(POSIX_MADV_DONTNEED) // QNX #define MADV_DONTNEED POSIX_MADV_DONTNEED #endif @@ -79,22 +92,25 @@ terms of the MIT license. A copy of the license can be found in the file #define MADV_FREE POSIX_MADV_FREE #endif -#define MI_UNIX_LARGE_PAGE_SIZE (2*MI_MiB) // todo: can we query the OS for this? +#define MI_UNIX_MAX_THP_PAGE_SIZE (8*MI_MiB) // above this we do not use madvise for transparent huge pages (see issue #1431) //------------------------------------------------------------------------------------ // Use syscalls for some primitives to allow for libraries that override open/read/close etc. // and do allocation themselves; using syscalls prevents recursion when mimalloc is // still initializing (issue #713) +// Not used on Android, see issue python/cpython/pull/158530 // Declare inline to avoid unused function warnings. //------------------------------------------------------------------------------------ -#if defined(MI_HAS_SYSCALL_H) && defined(SYS_open) && defined(SYS_close) && defined(SYS_read) && defined(SYS_access) +#if defined(MI_HAS_SYSCALL_H) && defined(SYS_open) && defined(SYS_close) && defined(SYS_read) && defined(SYS_access) && !defined(__ANDROID__) static inline int mi_prim_open(const char* fpath, int open_flags) { return syscall(SYS_open,fpath,open_flags,0); } static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) { - return syscall(SYS_read,fd,buf,bufsize); + ssize_t nread = syscall(SYS_read,fd,buf,bufsize); + if (nread > 0) { mi_msan_unpoison(buf, (size_t)nread); } + return nread; } static inline int mi_prim_close(int fd) { return syscall(SYS_close,fd); @@ -131,12 +147,13 @@ static bool unix_detect_overcommit(void) { #if defined(__linux__) int fd = mi_prim_open("/proc/sys/vm/overcommit_memory", O_RDONLY); if (fd >= 0) { - char buf[32]; - ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)); + char buf[64]; + ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)-1); mi_prim_close(fd); // // 0: heuristic overcommit, 1: always overcommit, 2: never overcommit (ignore NORESERVE) - if (nread >= 1) { + if (nread >= 1 && nread < (ssize_t)sizeof(buf)) { + buf[nread] = 0; os_overcommit = (buf[0] == '0' || buf[0] == '1'); } } @@ -158,12 +175,12 @@ static bool unix_detect_thp(void) { int fd = mi_prim_open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY); if (fd >= 0) { char buf[64]; - ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)); + ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)-1); mi_prim_close(fd); // // between brackets is the current value, for example: always [madvise] never - if (nread >= 1) { - if (nread > 64) { nread = 64; } + if (nread >= 1 && nread < (ssize_t)sizeof(buf)) { + buf[nread] = 0; thp_enabled = (_mi_strnstr(buf,nread,"[never]") == NULL); } } @@ -171,6 +188,27 @@ static bool unix_detect_thp(void) { return thp_enabled; } +static size_t unix_detect_large_page_size(void) { + size_t large_page_size = 2*MI_MiB; // assume 512*4KiB by default + #if defined(__linux__) + // get the transparent huge page size from the pmd size + const int fd = mi_prim_open("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", O_RDONLY); + if (fd >= 0) { + char buf[64]; + const ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)-1); + mi_prim_close(fd); + if (nread >= 1 && nread < (ssize_t)sizeof(buf)) { + buf[nread] = 0; + long size = 0; + if (_mi_strntol(buf, nread, NULL, &size) && size > 0 && (unsigned long)size <= SIZE_MAX) { + large_page_size = (size_t)size; + } + } + } + #endif + return large_page_size; +} + // try to detect the physical memory dynamically (if possible) static void unix_detect_physical_memory( size_t page_size, size_t* physical_memory_in_kib ) { #if defined(CTL_HW) && (defined(HW_PHYSMEM64) || defined(HW_MEMSIZE)) // freeBSD, macOS @@ -234,9 +272,10 @@ static size_t unix_detect_virtual_address_bits(void) { const int fd = mi_prim_open("/proc/cpuinfo", O_RDONLY); if (fd >= 0) { char buf[2048]; - const ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)); + const ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)-1); mi_prim_close(fd); - if ((nread >= 1) && (nread <= (ssize_t)sizeof(buf))) { + if ((nread >= 1) && (nread < (ssize_t)sizeof(buf))) { + buf[nread] = 0; if (_mi_strnstr(buf, nread, "sv39")) { return 39; } else if (_mi_strnstr(buf, nread, "sv48")) { return 48; } else if (_mi_strnstr(buf, nread, "sv57")) { return 57; } @@ -255,11 +294,11 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) config->alloc_granularity = (size_t)psize; unix_detect_physical_memory(config->page_size, &config->physical_memory_in_kib); } - config->large_page_size = MI_UNIX_LARGE_PAGE_SIZE; - config->has_overcommit = unix_detect_overcommit(); config->has_partial_free = true; // mmap can free in parts config->has_virtual_reserve = true; // todo: check if this true for NetBSD? (for anonymous mmap with PROT_NONE) - config->has_transparent_huge_pages = unix_detect_thp(); + config->large_page_size = unix_detect_large_page_size(); + config->has_transparent_huge_pages = (config->large_page_size <= MI_UNIX_MAX_THP_PAGE_SIZE) && unix_detect_thp(); + config->has_overcommit = unix_detect_overcommit(); config->virtual_address_bits = unix_detect_virtual_address_bits(); // disable transparent huge pages for this process? @@ -324,7 +363,7 @@ static void* unix_mmap_prim_aligned(void* addr, size_t size, size_t try_alignmen mi_bsr(try_alignment, &n); if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB p = unix_mmap_prim(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd); - if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { + if (p==MAP_FAILED || !mi_is_aligned(p,try_alignment)) { int err = errno; _mi_trace_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, addr); } @@ -345,7 +384,7 @@ static void* unix_mmap_prim_aligned(void* addr, size_t size, size_t try_alignmen void* hint = _mi_os_get_aligned_hint(try_alignment, size); if (hint != NULL) { p = unix_mmap_prim(hint, size, protect_flags, flags, fd); - if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { + if (p==MAP_FAILED || !mi_is_aligned(p,try_alignment)) { #if MI_TRACK_ENABLED // asan sometimes does not instrument errno correctly? int err = 0; #else @@ -460,30 +499,41 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec if (p == NULL) { *is_large = false; p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, flags, fd); - #if !defined(MI_NO_THP) - if (p != NULL && allow_large && mi_option_is_enabled(mi_option_allow_thp) && _mi_os_canuse_large_page(size, try_alignment)) { - #if defined(MADV_HUGEPAGE) - if (_mi_os_canuse_thp()) { - // Many Linux systems don't allow MAP_HUGETLB but they support instead - // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE - // though since properly aligned allocations will already use large pages if available - // in that case -- in particular for our large regions (in `memory.c`). - // However, some systems only allow THP if called with explicit `madvise`, so - // when large OS pages are enabled for mimalloc, we call `madvise` anyways. - if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) { - // *is_large = true; // possibly - }; + if (p!=NULL) { + #if defined(MADV_NOHUGEPAGE) + const size_t large_page_size = _mi_os_large_page_size(); + if (large_page_size > MI_UNIX_MAX_THP_PAGE_SIZE && size >= large_page_size) { + (void)unix_madvise(p, size, MADV_NOHUGEPAGE); // prevent using large transparent huge pages, see issue #1431 } - #elif defined(__sun) - struct memcntl_mha cmd = {0}; - cmd.mha_pagesize = _mi_os_large_page_size(); - cmd.mha_cmd = MHA_MAPSIZE_VA; - if (memcntl((caddr_t)p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) { - // *is_large = true; // possibly + else + #endif + { // keep the `else` above well-formed when MI_NO_THP is defined + #if !defined(MI_NO_THP) + if (allow_large && mi_option_is_enabled(mi_option_allow_thp) && _mi_os_canuse_large_page(size,try_alignment)) { + #if defined(MADV_HUGEPAGE) + if (_mi_os_has_thp()) { + // Many Linux systems don't allow MAP_HUGETLB but they support instead + // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE + // though since properly aligned allocations will already use large pages if available + // in that case -- in particular for our large regions (in `memory.c`). + // However, some systems only allow THP if called with explicit `madvise`, so + // when large OS pages are enabled for mimalloc, we call `madvise` anyways. + if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) { + // *is_large = true; // possibly + }; + } + #elif defined(__sun) + struct memcntl_mha cmd = {0}; + cmd.mha_pagesize = _mi_os_large_page_size(); + cmd.mha_cmd = MHA_MAPSIZE_VA; + if (memcntl((caddr_t)p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) { + // *is_large = true; // possibly + } + #endif } #endif + } } - #endif } return p; } @@ -783,7 +833,6 @@ mi_msecs_t _mi_prim_clock_now(void) { #if defined(__unix__) || defined(__unix) || defined(unix) || defined(__APPLE__) || defined(__HAIKU__) #include -#include #include #if defined(__APPLE__) @@ -969,6 +1018,7 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { static _Atomic(uintptr_t) no_getrandom; // = 0 if (mi_atomic_load_acquire(&no_getrandom)==0) { ssize_t ret = syscall(SYS_getrandom, buf, buf_len, GRND_NONBLOCK); + if (ret > 0) { mi_msan_unpoison(buf, (size_t)ret); } if (ret >= 0) return (buf_len == (size_t)ret); if (errno != ENOSYS) return false; mi_atomic_store_release(&no_getrandom, (uintptr_t)1); // don't call again, and fall back to /dev/urandom @@ -1014,7 +1064,8 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { // use pthread local storage keys to detect thread ending // (and used with MI_TLS_PTHREADS for the default theap) -pthread_key_t _mi_heap_default_key = MI_PTHREAD_KEY_INVALID; +// atomic as a thread may initialize while the process is exiting and deleting the key +static _Atomic(pthread_key_t) _mi_heap_default_key = MI_ATOMIC_VAR_INIT(MI_PTHREAD_KEY_INVALID); static void mi_pthread_done(void* value) { if (value!=NULL) { @@ -1023,21 +1074,24 @@ static void mi_pthread_done(void* value) { } void _mi_prim_thread_init_auto_done(void) { - mi_assert_internal(_mi_heap_default_key == MI_PTHREAD_KEY_INVALID); - pthread_key_create(&_mi_heap_default_key, &mi_pthread_done); + mi_assert_internal(mi_atomic_load_relaxed(&_mi_heap_default_key) == MI_PTHREAD_KEY_INVALID); + pthread_key_t key; + if (pthread_key_create(&key, &mi_pthread_done) == 0) { + mi_atomic_store_release(&_mi_heap_default_key, key); + } } void _mi_prim_thread_done_auto_done(void) { - pthread_key_t key = _mi_heap_default_key; + pthread_key_t key = mi_atomic_exchange_relaxed(&_mi_heap_default_key, MI_PTHREAD_KEY_INVALID); if (key != MI_PTHREAD_KEY_INVALID) { // do not leak the key, see issue #809 - _mi_heap_default_key = MI_PTHREAD_KEY_INVALID; pthread_key_delete(key); } } void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) { - if (_mi_heap_default_key != MI_PTHREAD_KEY_INVALID) { // can happen during recursive invocation on freeBSD - pthread_setspecific(_mi_heap_default_key, theap); + pthread_key_t key = mi_atomic_load_relaxed(&_mi_heap_default_key); + if (key != MI_PTHREAD_KEY_INVALID) { // can happen during recursive invocation on freeBSD + pthread_setspecific(key, theap); } } @@ -1065,3 +1119,7 @@ bool _mi_prim_thread_is_in_threadpool(void) { void _mi_prim_thread_yield(void) { sleep(0); } + +bool _mi_prim_process_is_killed(void) { + return false; // todo: can we detect when exit_group has been called and threads have been killed (potentially holding locks?) +} diff --git a/system/lib/mimalloc/src/prim/wasi/prim.c b/system/lib/mimalloc/src/prim/wasi/prim.c index 2d69b7245fb1a..1946cfc43409f 100644 --- a/system/lib/mimalloc/src/prim/wasi/prim.c +++ b/system/lib/mimalloc/src/prim/wasi/prim.c @@ -8,8 +8,10 @@ terms of the MIT license. A copy of the license can be found in the file // This file is included in `src/prim/prim.c` #include "mimalloc.h" -#include "mimalloc/internal.h" #include "mimalloc/prim.h" +#include "mimalloc/prim/os.h" +#include "mimalloc/util/libc.h" +#include "mimalloc/heap/init.h" #include // fputs #include // getenv @@ -53,7 +55,7 @@ int _mi_prim_free(void* addr, size_t size ) { } #elif defined(__wasi__) static void* mi_memory_grow( size_t size ) { - size_t base = (size > 0 ? __builtin_wasm_memory_grow(0,_mi_divide_up(size, _mi_os_page_size())) + size_t base = (size > 0 ? __builtin_wasm_memory_grow(0,mi_divide_up(size, _mi_os_page_size())) : __builtin_wasm_memory_size(0)); if (base == SIZE_MAX) return NULL; return (void*)(base * _mi_os_page_size()); @@ -88,8 +90,8 @@ static void* mi_prim_mem_grow(size_t size, size_t try_alignment) { { void* current = mi_memory_grow(0); // get current size if (current != NULL) { - void* aligned_current = _mi_align_up_ptr(current, try_alignment); // and align from there to minimize wasted space - alloc_size = _mi_align_up( ((uint8_t*)aligned_current - (uint8_t*)current) + size, _mi_os_page_size()); + void* aligned_current = mi_align_up_ptr(current, try_alignment); // and align from there to minimize wasted space + alloc_size = mi_align_up( ((uint8_t*)aligned_current - (uint8_t*)current) + size, _mi_os_page_size()); base = mi_memory_grow(alloc_size); } } @@ -97,7 +99,7 @@ static void* mi_prim_mem_grow(size_t size, size_t try_alignment) { pthread_mutex_unlock(&mi_theap_grow_mutex); #endif if (base != NULL) { - p = _mi_align_up_ptr(base, try_alignment); + p = mi_align_up_ptr(base, try_alignment); if ((uint8_t*)p + size > (uint8_t*)base + alloc_size) { // another thread used wasm_memory_grow/sbrk in-between and we do not have enough // space after alignment. Give up (and waste the space as we cannot shrink :-( ) @@ -188,7 +190,7 @@ size_t _mi_prim_numa_node_count(void) { // low resolution timer static mi_msecs_t mi_prim_clock_now_lowres(void) { const int64_t ticks = (int64_t)clock(); - #if !defined(CLOCKS_PER_SEC) + #if !defined(CLOCKS_PER_SEC) return ticks; #else if (CLOCKS_PER_SEC <= 0 || CLOCKS_PER_SEC == 1000) { @@ -210,12 +212,12 @@ mi_msecs_t _mi_prim_clock_now(void) { #else const clockid_t clockid = CLOCK_REALTIME; #endif - struct timespec t; + struct timespec t; if (clock_gettime(clockid,&t) == 0) { return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000L); } - #endif - return mi_prim_clock_now_lowres(); + #endif + return mi_prim_clock_now_lowres(); } @@ -295,3 +297,7 @@ bool _mi_prim_thread_is_in_threadpool(void) { void _mi_prim_thread_yield(void) { sleep(0); } + +bool _mi_prim_process_is_killed(void) { + return false; +} diff --git a/system/lib/mimalloc/src/prim/windows/prim.c b/system/lib/mimalloc/src/prim/windows/prim.c index 222fa6303e496..4eb1dde4577bb 100644 --- a/system/lib/mimalloc/src/prim/windows/prim.c +++ b/system/lib/mimalloc/src/prim/windows/prim.c @@ -8,8 +8,11 @@ terms of the MIT license. A copy of the license can be found in the file // This file is included in `src/prim/prim.c` #include "mimalloc.h" -#include "mimalloc/internal.h" #include "mimalloc/prim.h" +#include "mimalloc/prim/os.h" +#include "mimalloc/util/libc.h" +#include "mimalloc/util/options.h" +#include "mimalloc/heap/init.h" #include // fputs, stderr #include // atexit @@ -71,12 +74,14 @@ typedef BOOL (__stdcall* PGetNumaNodeProcessorMaskEx)(USHORT Node, PGROUP_AFFINI typedef BOOL (__stdcall *PGetNumaProcessorNode)(UCHAR Processor, PUCHAR NodeNumber); typedef BOOL (__stdcall* PGetNumaNodeProcessorMask)(UCHAR Node, PULONGLONG ProcessorMask); typedef BOOL (__stdcall* PGetNumaHighestNodeNumber)(PULONG Node); +typedef BOOL (__stdcall* PRtlDllShutdownInProgress)(VOID); static PGetCurrentProcessorNumberEx pGetCurrentProcessorNumberEx = NULL; static PGetNumaProcessorNodeEx pGetNumaProcessorNodeEx = NULL; static PGetNumaNodeProcessorMaskEx pGetNumaNodeProcessorMaskEx = NULL; static PGetNumaProcessorNode pGetNumaProcessorNode = NULL; static PGetNumaNodeProcessorMask pGetNumaNodeProcessorMask = NULL; static PGetNumaHighestNodeNumber pGetNumaHighestNodeNumber = NULL; +static PRtlDllShutdownInProgress pRtlDllShutdownInProgress = NULL; // Not available on xbox typedef SIZE_T(__stdcall* PGetLargePageMinimum)(VOID); @@ -88,19 +93,19 @@ typedef BOOL (__stdcall* PGetVersionExW)(LPOSVERSIONINFOW lpVersionInformation); // Load a library -static HMODULE mi_win_loadlibrary(const TCHAR* library) { +static HMODULE mi_win_loadlibrary(const wchar_t* library) { #if MI_WIN_DESKTOP - return LoadLibrary(library); + return LoadLibraryW(library); #else return LoadPackagedLibrary(library, 0); #endif } // Get a library handle (and possibly load it) -static HMODULE mi_win_getlibrary(const TCHAR* library, bool* should_free) { +static HMODULE mi_win_getlibrary(const wchar_t* library, bool* should_free) { #if MI_WIN_DESKTOP // avoid calling LoadLibrary for "kernel32", "ntdll", and "kernelbase" (also to avoid hitting the loader lock) - HMODULE mod = GetModuleHandle(library); + HMODULE mod = GetModuleHandleW(library); if (mod!=NULL) { *should_free = false; return mod; @@ -195,7 +200,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) // get the VirtualAlloc2 function bool hDllFree; - HINSTANCE hDll = mi_win_getlibrary(TEXT("kernelbase.dll"), &hDllFree); + HINSTANCE hDll = mi_win_getlibrary(L"kernelbase.dll", &hDllFree); if (hDll != NULL) { // use VirtualAlloc2FromApp if possible as it is available to Windows store apps pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2FromApp"); @@ -203,13 +208,14 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) mi_win_freelibrary(hDll, hDllFree); } // NtAllocateVirtualMemoryEx is used for huge page allocation - hDll = mi_win_getlibrary(TEXT("ntdll.dll"), &hDllFree); + hDll = mi_win_getlibrary(L"ntdll.dll", &hDllFree); if (hDll != NULL) { pNtAllocateVirtualMemoryEx = (PNtAllocateVirtualMemoryEx)(void (*)(void))GetProcAddress(hDll, "NtAllocateVirtualMemoryEx"); + pRtlDllShutdownInProgress = (PRtlDllShutdownInProgress)(void (*)(void))GetProcAddress(hDll, "RtlDllShutdownInProgress"); mi_win_freelibrary(hDll, hDllFree); } // Try to use Win7+ numa API - hDll = mi_win_getlibrary(TEXT("kernel32.dll"), &hDllFree); + hDll = mi_win_getlibrary(L"kernel32.dll", &hDllFree); if (hDll != NULL) { pGetCurrentProcessorNumberEx = (PGetCurrentProcessorNumberEx)(void (*)(void))GetProcAddress(hDll, "GetCurrentProcessorNumberEx"); pGetNumaProcessorNodeEx = (PGetNumaProcessorNodeEx)(void (*)(void))GetProcAddress(hDll, "GetNumaProcessorNodeEx"); @@ -439,8 +445,13 @@ int _mi_prim_reuse(void* addr, size_t size) { } int _mi_prim_protect(void* addr, size_t size, bool protect) { + #if !MI_WIN_DESKTOP && WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) + ULONG oldprotect = 0; + BOOL ok = VirtualProtectFromApp(addr, size, protect ? PAGE_NOACCESS : PAGE_READWRITE, &oldprotect); + #else DWORD oldprotect = 0; BOOL ok = VirtualProtect(addr, size, protect ? PAGE_NOACCESS : PAGE_READWRITE, &oldprotect); + #endif return (ok ? 0 : (int)GetLastError()); } @@ -608,7 +619,7 @@ void _mi_prim_process_info(mi_process_info_t* pinfo) // load psapi on demand mi_atomic_do_once{ - HINSTANCE hDll = mi_win_loadlibrary(TEXT("psapi.dll")); + HINSTANCE hDll = mi_win_loadlibrary(L"psapi.dll"); if (hDll != NULL) { pGetProcessMemoryInfo = (PGetProcessMemoryInfo)(void (*)(void))GetProcAddress(hDll, "GetProcessMemoryInfo"); // mi_win_freelibrary(hDll, true); // don't free @@ -644,10 +655,6 @@ void _mi_prim_out_stderr( const char* msg ) if (hcon == INVALID_HANDLE_VALUE) { hcon = GetStdHandle(STD_ERROR_HANDLE); // returns NULL if no stderr is available #if MI_WIN_DESKTOP - if (hcon==NULL) { - AttachConsole(ATTACH_PARENT_PROCESS); // if started from a parent console, try to attach to that - hcon = GetStdHandle(STD_ERROR_HANDLE); - } CONSOLE_SCREEN_BUFFER_INFO sbi; hconIsConsole = ((hcon != NULL && hcon != INVALID_HANDLE_VALUE) && GetConsoleScreenBufferInfo(hcon, &sbi)); #endif @@ -667,6 +674,13 @@ void _mi_prim_out_stderr( const char* msg ) WriteFile(hcon, msg, (DWORD)len, &written, NULL); } } + #if MI_WIN_DESKTOP + else if (hcon == NULL) { + // no stderr handle: this is usually a GUI or detached process. Send the + // output to the debugger instead (see issue #1354 and #1429) + OutputDebugStringA(msg); + } + #endif // don't fall back to fputs or _cputs as the crt can have it locked } } @@ -692,41 +706,63 @@ int _mi_prim_getenv(const char* name, char* result, size_t result_size) { // Random //---------------------------------------------------------------- -#if defined(MI_USE_RTLGENRANDOM) // || defined(__cplusplus) -// We prefer to use BCryptGenRandom instead of (the unofficial) RtlGenRandom but when using -// dynamic overriding, we observed it can raise an exception when compiled with C++, and -// sometimes deadlocks when also running under the VS debugger. -// In contrast, issue #623 implies that on Windows Server 2019 we need to use BCryptGenRandom. -// To be continued.. +#if defined(MI_USE_RTLGENRANDOM) || MI_WIN_DESKTOP +#if defined(_MSC_VER) #pragma comment (lib,"advapi32.lib") +#endif #define RtlGenRandom SystemFunction036 mi_decl_externc BOOLEAN NTAPI RtlGenRandom(PVOID RandomBuffer, ULONG RandomBufferLength); -bool _mi_prim_random_buf(void* buf, size_t buf_len) { +static bool mi_win_rtl_gen_random(void* buf, size_t buf_len) { + mi_assert(buf_len <= ULONG_MAX); + if (buf_len > ULONG_MAX) return false; return (RtlGenRandom(buf, (ULONG)buf_len) != 0); } +#endif -#else +#if defined(MI_USE_RTLGENRANDOM) +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + return mi_win_rtl_gen_random(buf, buf_len); +} +#elif !MI_WIN_DESKTOP +// UWP/Xbox cannot dynamically load system libraries, so we link to BCryptGenRandom directly. +#include +#if defined(_MSC_VER) +#pragma comment (lib,"bcrypt.lib") +#endif #ifndef BCRYPT_USE_SYSTEM_PREFERRED_RNG #define BCRYPT_USE_SYSTEM_PREFERRED_RNG 0x00000002 #endif -typedef LONG (NTAPI *PBCryptGenRandom)(HANDLE, PUCHAR, ULONG, ULONG); -static PBCryptGenRandom pBCryptGenRandom = NULL; - bool _mi_prim_random_buf(void* buf, size_t buf_len) { mi_assert(buf_len <= ULONG_MAX); if (buf_len > ULONG_MAX) return false; + return (BCryptGenRandom(NULL, (PUCHAR)buf, (ULONG)buf_len, BCRYPT_USE_SYSTEM_PREFERRED_RNG) >= 0); +} + +#else +// We use the documented ProcessPrng (Windows 8+) which always succeeds, and fall back to RtlGenRandom +// if it is not available. We avoid BCryptGenRandom as on Windows 7 / Server 2008 R2 it seems it can deadlock +// when called while the loader lock is held (as during process attach) (issues #1395 and #1426). +typedef BOOL (WINAPI *PProcessPrng)(PBYTE pbData, SIZE_T cbData); +static PProcessPrng pProcessPrng = NULL; + +bool _mi_prim_random_buf(void* buf, size_t buf_len) { mi_atomic_do_once { - HINSTANCE hDll = mi_win_loadlibrary(TEXT("bcrypt.dll")); - if (hDll != NULL) { - pBCryptGenRandom = (PBCryptGenRandom)(void (*)(void))GetProcAddress(hDll, "BCryptGenRandom"); - // mi_win_freelibrary(hDll); // don't free + if (win_major_version > 6 || (win_major_version == 6 && win_minor_version >= 2)) { + bool should_free = false; + HMODULE hDll = mi_win_getlibrary(L"bcryptprimitives.dll", &should_free); + if (hDll != NULL) { + pProcessPrng = (PProcessPrng)(void (*)(void))GetProcAddress(hDll, "ProcessPrng"); + if (pProcessPrng == NULL) { mi_win_freelibrary(hDll, should_free); } // otherwise don't free + } } } - if (pBCryptGenRandom == NULL) return false; - return (pBCryptGenRandom(NULL, (PUCHAR)buf, (ULONG)buf_len, BCRYPT_USE_SYSTEM_PREFERRED_RNG) >= 0); + if (pProcessPrng != NULL) { + return (pProcessPrng((PBYTE)buf, buf_len) != FALSE); + } + return mi_win_rtl_gen_random(buf, buf_len); } #endif // MI_USE_RTLGENRANDOM @@ -753,6 +789,15 @@ void _mi_prim_thread_yield(void) { SwitchToThread(); } +bool _mi_prim_process_is_killed(void) { + if (pRtlDllShutdownInProgress != NULL) { + return (*pRtlDllShutdownInProgress)(); + } + else { + return false; + } +} + //---------------------------------------------------------------- // Process & Thread Init/Done //---------------------------------------------------------------- @@ -792,7 +837,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { #if defined(MI_WIN_INIT_USE_CRT_TLS) #if !defined(__MINGW32__) || defined(MI_MINGW_UCRT64) // on mingw without UCRT use the constructor attribute (in `src/prim/prim.c`) - #define MI_PRIM_HAS_PROCESS_ATTACH 1 + #define MI_PRIM_HAS_PROCESS_ATTACH 1 #endif // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event. @@ -809,10 +854,20 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { return ((imageNtHeaders->FileHeader.Characteristics & IMAGE_FILE_DLL) == IMAGE_FILE_DLL); } +#if defined(__cplusplus) +extern "C" { +#endif + extern IMAGE_DOS_HEADER __ImageBase; // supplied by the linker +#if defined(__cplusplus) +} +#endif + static bool mi_current_module_is_dll(void) { - HMODULE mod = NULL; - const BOOL ok = GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, (LPCSTR)mi_current_module_is_dll, &mod); - return (ok && mi_module_is_dll(mod)); + // HMODULE mod = NULL; + // const BOOL ok = GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, (LPCSTR)mi_current_module_is_dll, &mod); + // return (ok && mi_module_is_dll(mod)); + const HMODULE mod = (HMODULE)(&__ImageBase); + return mi_module_is_dll(mod); } // Hook into CRT initialization and finalization. @@ -868,7 +923,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { typedef int (mi_cdecl* mi_crt_callback_t)(void); - #if defined(_WIN64) && defined(_MSC_VER) // 64-bit + #if (defined(_WIN64) || defined(_M_ARM)) && defined(_MSC_VER) // no underscore #pragma comment(linker, "/INCLUDE:_tls_used") #pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre") #pragma comment(linker, "/INCLUDE:_mi_tls_callback_post") @@ -885,7 +940,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { extern const mi_crt_callback_t _mi_crt_callback_init[]; const mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init }; #pragma const_seg() - #elif defined(_MSC_VER) // 32-bit + #elif defined(_MSC_VER) // x86 32-bit, add underscore #pragma comment(linker, "/INCLUDE:__tls_used") #pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre") #pragma comment(linker, "/INCLUDE:__mi_tls_callback_post") @@ -974,7 +1029,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { extern "C" { #endif - #if defined(_WIN64) && defined(_MSC_VER) // 64-bit + #if (defined(_WIN64) || defined(_M_ARM)) && defined(_MSC_VER) // no underscore #pragma comment(linker, "/INCLUDE:_tls_used") #pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre") #pragma comment(linker, "/INCLUDE:_mi_tls_callback_post") @@ -986,7 +1041,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[]; const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach }; #pragma const_seg() - #elif defined(_MSC_VER) // 32-bit + #elif defined(_MSC_VER) // x86 32-bit #pragma comment(linker, "/INCLUDE:__tls_used") #pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre") #pragma comment(linker, "/INCLUDE:__mi_tls_callback_post") @@ -1048,7 +1103,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { extern "C" { #endif - #if defined(_WIN64) && defined(_MSC_VER) // 64-bit + #if (defined(_WIN64) || defined(_M_ARM)) && defined(_MSC_VER) // no underscore #pragma comment(linker, "/INCLUDE:_tls_used") #pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre") #pragma comment(linker, "/INCLUDE:_mi_tls_callback_post") @@ -1060,7 +1115,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[]; const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach }; #pragma const_seg() - #elif defined(_MSC_VER) // 32-bit + #elif defined(_MSC_VER) // x86 32-bit #pragma comment(linker, "/INCLUDE:__tls_used") #pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre") #pragma comment(linker, "/INCLUDE:__mi_tls_callback_post") @@ -1073,8 +1128,8 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { #elif defined(__MINGW32__) extern const IMAGE_TLS_DIRECTORY _tls_used; __attribute__((used)) static const void* const mi_tls_used_ref = &_tls_used; // pull in the CRT tls - __attribute__((used, section(".CRT$XLB"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_pre = &mi_tls_attach; - __attribute__((used, section(".CRT$XLY"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_post = &mi_tls_detach; + __attribute__((used, section(".CRT$XLB"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_pre = &mi_win_main_attach; + __attribute__((used, section(".CRT$XLY"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_post = &mi_win_main_detach; #endif #if defined(__cplusplus) @@ -1199,4 +1254,3 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { mi_allocator_done(); } #endif - diff --git a/system/lib/mimalloc/src/prim/windows/readme.md b/system/lib/mimalloc/src/prim/windows/readme.md index 217c3d174db47..a3c0619113508 100644 --- a/system/lib/mimalloc/src/prim/windows/readme.md +++ b/system/lib/mimalloc/src/prim/windows/readme.md @@ -13,5 +13,5 @@ > wpr -start src\prim\windows\etw-mimalloc.wprp -filemode > > wpr -stop test.etl - ``` - and then open `test.etl` in the Windows Performance Analyzer (WPA). \ No newline at end of file + ``` + and then open `test.etl` in the Windows Performance Analyzer (WPA). diff --git a/system/lib/mimalloc/src/profile/pprof.c b/system/lib/mimalloc/src/profile/pprof.c new file mode 100644 index 0000000000000..4b925f38ced9f --- /dev/null +++ b/system/lib/mimalloc/src/profile/pprof.c @@ -0,0 +1,1538 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. + +Disclaimer: Unlike the rest of the considerate hand-crafted mimalloc sources, +this file was written with AI assistance (for the process image and protobuf +format generation). +-----------------------------------------------------------------------------*/ + +// --------------------------------------------------------------------------- +// This implements a basic memory profiler that outputs pprof-compatible heap +// snapshots (text or protobuf). It is basic, but it does scale with a lock-free +// hash table for locations, and uses poisson distributed sampling. +// It has no impact on runtime performance when profiling is not active. +// +// It can be used programmatically (using `mi_pprof_profiler_new` etc.) +// or be invoked with environment variables. For example: +// MIMALLOC_PROFILE=profile ./my_program +// and then analyze the generated profile using pprof tools: +// pprof -http=:8080 ./myprogram profile.* +// --------------------------------------------------------------------------- + +#include "mimalloc.h" +#include "mimalloc/prim/prim-tls.h" // _mi_theap_default +#include "mimalloc/util/libc.h" +#include "mimalloc/util/options.h" +#include "mimalloc/util/stats.h" +#include "mimalloc/profile/pprof.h" + +// --------------------------------------------------------------------------- +// Internal API +// --------------------------------------------------------------------------- + +typedef struct mi_location_s mi_location_t; +typedef struct mi_locations_s mi_locations_t; +typedef struct mi_callstack_s mi_callstack_t; + +static bool mi_locations_init(mi_heap_t* heap, mi_locations_t* locations); +static void mi_locations_done(mi_heap_t* heap, mi_locations_t* locations); +static mi_location_t* mi_locations_find_or_insert(mi_heap_t* heap, mi_locations_t* locations, mi_threadid_t thread_id, const mi_callstack_t* callstack, size_t callstack_hash); + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- +#define MI_MAX_BACKTRACE_DEPTH (100) +#define MI_BACKTRACE_SKIP_FRAMES (3) + +// Threshold (in bytes) for the thread-local interval countdown batching in `on_alloc` +#ifndef MI_PROFILE_COUNTDOWN_THRESHOLD +#define MI_PROFILE_COUNTDOWN_THRESHOLD (8*MI_KiB) +#endif +#define MI_PROFILE_COUNTDOWN_MIN_INTERVAL (8*MI_PROFILE_COUNTDOWN_THRESHOLD) + +// A callstack represents a sequence of function call frames captured at a specific point in time. +struct mi_callstack_s { + size_t count; + void** frames; +}; + +// A location is a unique combination of a thread and runtime callstack for +// which we track allocation and deallocation statistics. +// +// Locations are only ever inserted, or mutated in-place after publication, and a location is only ever looked up using its own +// `hash`/`thread_id`/`callstack` which allows a lock-free lookup of locations in the hash table. +struct mi_location_s { + _Atomic(struct mi_location_s*) next; // next location in the same hash bucket (for chaining on collision); + mi_threadid_t thread_id; + mi_callstack_t callstack; + size_t hash; // hash of the thread_id and callstack + int64_t alloc_count; // only ever written by the single (allocating) owning thread: not atomic + int64_t alloc_bytes; + _Atomic(size_t) inuse_count; // currently in-use (allocated but not yet freed) objects/bytes for this location; + _Atomic(size_t) inuse_bytes; // incremented in `on_alloc` and decremented in `on_free`, both possibly on any thread +}; + +// A basic hash table of locations. The bucket array is fixed-size (allocated +// once in `mi_locations_init`) and each bucket is a lock-free singly-linked +// list that only ever grows by prepending (see `mi_locations_find_or_insert`). +struct mi_locations_s { + size_t bucket_count; // number of buckets (fixed size) + _Atomic(mi_location_t*)* buckets; // array of `bucket_count` chain heads + _Atomic(size_t) count; // number of locations currently stored (just a statistic) +}; + +// Our profiler. +typedef struct { + mi_profiler_t profiler; // mimalloc profiler info + mi_heap_t* profile_heap; // heap used for all profiler allocations (so these don't interfere) + mi_locations_t locations; // hash table of call locations + size_t sample_threshold; // current sample threshold + size_t snapshot_count; // number of times `mi_profiler_snapshot` was called (used to number the snapshot files) + char* base_file_name; // base file name for snapshot files, e.g. "..heap" + bool format_text; // text or protobuf snapshot format + size_t alloc_interval_size; // if >0, automatically snapshot every `alloc_interval_size` allocated bytes (capped at `MI_SSIZE_MAX`) + _Atomic(mi_ssize_t) alloc_interval_countdown; // bytes remaining until the next automatic snapshot (can go negative; decremented concurrently by `on_alloc` on any thread) + size_t inuse_interval_size; // if >0, automatically snapshot every time (sampled) in-use bytes grow by `inuse_interval_size` bytes + _Atomic(size_t) inuse_bytes; // running total of (sampled) in-use bytes across all locations (can never exceed the address space); incremented in `on_alloc`, decremented in `on_free`, both possibly on any thread + _Atomic(size_t) inuse_interval_threshold; // next `inuse_bytes` total that triggers a snapshot; advanced (monotonically increasing) whenever reached in `on_alloc` + size_t time_interval_secs; // if >0, automatically snapshot every `time_interval_secs` seconds (checked from `on_alloc`) + _Atomic(mi_msecs_t) time_interval_deadline; // next `_mi_clock_now()` value that triggers a snapshot; advanced (monotonically increasing) whenever reached in `on_alloc` + int64_t start_time_secs; // wall-clock time at creation (seconds since the Unix epoch) + mi_msecs_t start_clock; // `_mi_clock_now()` at creation (monotonic, so snapshot times are `start_time_secs` + elapsed clock) +} mi_pprof_profiler_t; + +static inline mi_pprof_profiler_t* downcast( mi_profiler_t* prof ) { + return (mi_pprof_profiler_t*)prof; +} + +static void mi_pprof_profiler_snapshot(mi_pprof_profiler_t* prof); +static size_t mi_prim_backtrace(void** buffer, size_t max_depth, size_t* hash); // `buffer` must have room for `max_depth + MI_PPROF_SKIP_FRAMES + 1` entries (see definition) +static int64_t mi_prim_time_secs(void); // wall-clock seconds since the Unix epoch (0 if unavailable) + + +// --------------------------------------------------------------------------- +// Automatic profiler driven by the `MIMALLOC_PROFILE` environment variable. +// This allows profiling an arbitrary program (with no code changes) by +// overriding its allocator with a mimalloc shared library build and setting +// `MIMALLOC_PROFILE=` +// (and optionally `MIMALLOC_PROFILE_ALLOC_INTERVAL=` to periodically snapshot +// while the program is running). +// --------------------------------------------------------------------------- +#ifndef MI_PROFILE_USE_BUILTIN +#define MI_PROFILE_USE_BUILTIN (MI_PROFILE) +#endif + +#if MI_PROFILE_USE_BUILTIN // todo: also disable profiling in secure mode? + +static bool mi_is_elevated_process(void); +static mi_profiler_t* mi_profile_env_profiler; // NULL if `MIMALLOC_PROFILE` was not set (or initialization failed) + +// Called once at process initialization (see `mi_process_init` in `heap/init.c`). +void _mi_pprof_profiler_init(void) { + #if MI_PROFILE + if (mi_is_elevated_process()) return; // don't let an untrusted environment influence a privileged process + char fname[1024]; + if (_mi_getenv("mimalloc_profile", fname, sizeof(fname)) != 0 || fname[0] == 0) return; // `MIMALLOC_PROFILE` not set + const size_t sample_rate = mi_option_get_size(mi_option_profile_sample_rate); // 512 KiB by default + const size_t alloc_interval_size = mi_option_get_size(mi_option_profile_alloc_interval); // 1 GiB by default + const size_t inuse_interval_size = mi_option_get_size(mi_option_profile_inuse_interval); // 0 by default (no automatic interval snapshots) + const size_t time_interval_secs = mi_option_get_clamp(mi_option_profile_time_interval,0,24*60*60L); // 0 by default (no automatic interval snapshots) + mi_profiler_t* profiler = mi_pprof_profiler_new(sample_rate, fname, alloc_interval_size, inuse_interval_size, time_interval_secs); + if (profiler == NULL) return; + mi_profile_env_profiler = profiler; + _mi_verbose_message("pprof profiler initialized with base file name: %s\n", fname); + mi_profile(profiler); // attach to the main sub-process (and any current/future heaps in it) + mi_profiler_start(profiler); + #endif +} + +// Called once at process termination (see `mi_process_done` in `heap/init.c`). +void _mi_pprof_profiler_done(void) { + #if MI_PROFILE + mi_profiler_t* profiler = mi_profile_env_profiler; + if (profiler == NULL) return; + mi_profile_env_profiler = NULL; + mi_profiler_stop(profiler); + _mi_verbose_message("pprof profiler done\n"); + mi_profiler_snapshot(profiler); // final snapshot so short-lived processes still get a complete profile + mi_profile(NULL); // detach from the main sub-process + mi_pprof_profiler_delete(profiler); + #endif +} + +#else + +void _mi_pprof_profiler_init(void) { +} + +void _mi_pprof_profiler_done(void) { +} + +#endif + +// --------------------------------------------------------------------------- +// Get a location +// --------------------------------------------------------------------------- + +static mi_decl_noinline mi_location_t* mi_location_get(mi_pprof_profiler_t* prof) { + void* frames[MI_MAX_BACKTRACE_DEPTH + 1 + MI_BACKTRACE_SKIP_FRAMES]; // extra room: `mi_prim_backtrace` captures+shifts out its own frame and `MI_BACKTRACE_SKIP_FRAMES` wrapper frames in-place + size_t callstack_hash = 0; + const size_t depth = mi_prim_backtrace(frames, MI_MAX_BACKTRACE_DEPTH, &callstack_hash); + if (depth==0) return NULL; + mi_callstack_t callstack = { depth, frames }; + const mi_threadid_t thread_id = _mi_thread_id(); + mi_location_t* loc = mi_locations_find_or_insert(prof->profile_heap, &prof->locations, thread_id, &callstack, callstack_hash); + return loc; +} + +// --------------------------------------------------------------------------- +// Profiler callbacks and initialization +// --------------------------------------------------------------------------- + +// Draws an integer approximating Exp(scale) to give a poisson distribution, +// instead of a fixed period, which avoids sampling bias from allocation +// patterns that happen to align with a constant period +static size_t mi_exp_sample(mi_theap_t* theap, size_t scale) { + if (scale == 0) return 0; // keep 0 + #if MI_PROFILE==2 + if (scale == 1) return 1; // keep 1 as is (to allow sampling every allocation) + #endif + const size_t r = _mi_theap_random_next(theap) | 1; + const size_t lz = mi_clz(r); // coarse -log2(U): Geometric(1/2) + // The highest set bit of `r` is at position `MI_SIZE_BITS-1-lz`; shift it out and + // take the next 8 bits as a fractional refinement in [0,256). When `r == 1` + // (lz == MI_SIZE_BITS-1) there are no fractional bits and the shift would be + // undefined, so the fraction is 0 there. + const size_t frac_q8 = (lz >= MI_SIZE_BITS - 1 ? 0 : (r << (lz + 1)) >> (MI_SIZE_BITS - 8)); + const size_t bits_q8 = (lz << 8) | frac_q8; // Q8 fixed-point estimate of -log2(U) + const size_t ln_q8 = (bits_q8 * 177) >> 8; // -log2(U) -> -ln(U): *ln(2) (~177/256) + size_t next = ((ln_q8 * scale) >> 8) + 1; // scale by the mean; +1 avoids a zero threshold + const size_t cap = scale * 32; // clamp the (rare) long tail + return (next > cap ? cap : next); +} + +// Check if the sampled bytes accumulated so far (batched per-thread) should trigger a shared +// countdown decrement, and if so, whether that decrement crosses zero (triggering a snapshot and +// resetting the countdown to `alloc_interval_size`). +static bool mi_interval_update(uint64_t ubytes_since_last_sample, size_t alloc_interval_size, _Atomic(mi_ssize_t)* alloc_interval_countdown, mi_ssize_t* interval_pending) { + mi_ssize_t sampled_bytes = 0; + mi_ssize_t bytes_since_last_sample= (ubytes_since_last_sample > MI_SSIZE_MAX ? MI_SSIZE_MAX : (mi_ssize_t)ubytes_since_last_sample); + if (alloc_interval_size < MI_PROFILE_COUNTDOWN_MIN_INTERVAL || MI_SSIZE_MAX - *interval_pending < bytes_since_last_sample /* overflow? */) { + // interval too small relative to the batching; always adjust + sampled_bytes = bytes_since_last_sample; + } + else { + // batch locally per-thread and only touch the shared (contended) countdown + // once the local pending amount reaches `MI_PROFILE_COUNTDOWN_THRESHOLD`. + *interval_pending += bytes_since_last_sample; + if (*interval_pending >= (mi_ssize_t)MI_PROFILE_COUNTDOWN_THRESHOLD) { + sampled_bytes = *interval_pending; + *interval_pending = 0; + } + } + // update profile interval countdown if we have sampled bytes + if (sampled_bytes > 0) { + const mi_ssize_t new_countdown = mi_atomic_addi_relaxed(alloc_interval_countdown, -sampled_bytes) - sampled_bytes; + if (new_countdown <= 0 && new_countdown + sampled_bytes > 0) { + mi_atomic_storess_relaxed(alloc_interval_countdown, (mi_ssize_t)alloc_interval_size); + return true; + } + } + return false; +} + +// Check if the (global) in-use byte total has reached `inuse_interval_threshold` +static bool mi_inuse_threshold_update(size_t new_total, size_t inuse_interval_size, _Atomic(size_t)* inuse_interval_threshold) { + size_t threshold = mi_atomic_load_relaxed(inuse_interval_threshold); + while (new_total >= threshold) { + size_t next_threshold = threshold + inuse_interval_size; + while (next_threshold <= new_total) { next_threshold += inuse_interval_size; } // skip ahead if we grew by more than one interval + if (mi_atomic_cas_weak_relaxed(inuse_interval_threshold, &threshold, next_threshold)) { + return true; + } + // CAS failed: `threshold` now holds the current value; retry (another thread may have already advanced it far enough) + } + return false; +} + +// Check if `now` has reached `time_interval_deadline`, and if so, atomically advance the +// deadline to the next multiple of `time_interval_secs` beyond `now`, returning `true` (to the +// single thread that won the race) so a snapshot is triggered exactly once. +static bool mi_time_threshold_update(mi_msecs_t now, size_t time_interval_secs, _Atomic(mi_msecs_t)* time_interval_deadline) { + const mi_msecs_t interval_msecs = (mi_msecs_t)time_interval_secs * 1000; + mi_msecs_t deadline = mi_atomic_loadi64_relaxed(time_interval_deadline); + while (now >= deadline) { + mi_msecs_t next_deadline = deadline + interval_msecs; + while (next_deadline <= now) { next_deadline += interval_msecs; } // skip ahead if more than one interval has passed + if (mi_atomic_casi64_strong_acq_rel(time_interval_deadline, &deadline, next_deadline)) { + return true; + } + // CAS failed: `deadline` now holds the current value; retry (another thread may have already advanced it far enough) + } + return false; +} + +// Sample allocation event and update profiling data accordingly +static size_t mi_cdecl on_alloc(mi_profiler_t* profiler, mi_profiler_sample_data_t* data, void* ptr, size_t requested_size, size_t threshold, uint64_t bytes_since_last_sample, mi_heap_t* heap) +{ + MI_UNUSED(threshold); MI_UNUSED(ptr); + mi_pprof_profiler_t* prof = downcast(profiler); + mi_theap_t* theap = _mi_heap_theap(heap); + mi_location_t* loc = mi_location_get(prof); // nested allocations (e.g. by `backtrace`) are not sampled (see `mi_profiler_on_alloc`) + const size_t new_threshold = mi_exp_sample(theap, prof->sample_threshold); // randomized (Poisson) next sample threshold + const size_t alloc_size = requested_size; // or mi_heap_usable_size(heap,ptr) ? + + if (data!=NULL) { + // pass the location to the corresponding mi_free (maybe called from another thread) + mi_assert(data->user_data_size >= 2*sizeof(void*)); + data->user_data[0] = loc; + data->user_data[1] = (void*)((uintptr_t)alloc_size); + } + if (loc!=NULL) { + // `on_alloc` for a given location only ever runs on its one owning (allocating) + // thread, so plain increments are safe here (no concurrent writers possible). + loc->alloc_bytes += (int64_t)alloc_size; + loc->alloc_count += 1; + // `inuse_count`/`inuse_bytes` can be decremented concurrently by `on_free` on any + // thread, so these updates must be atomic. + mi_atomic_add_relaxed(&loc->inuse_bytes, alloc_size); + mi_atomic_increment_relaxed(&loc->inuse_count); + } + + if (prof->alloc_interval_size > 0) { // interval-based profiling enabled + mi_ssize_t* pinterval_pending = (mi_ssize_t*)&theap->profiler_reserved1; + if (mi_interval_update(bytes_since_last_sample, prof->alloc_interval_size, &prof->alloc_interval_countdown, pinterval_pending)) { + mi_profiler_snapshot(profiler); + } + } + if (prof->inuse_interval_size > 0) { // snapshot whenever (sampled) in-use memory has grown by `inuse_interval_size` bytes + const size_t total_inuse_bytes = mi_atomic_add_relaxed(&prof->inuse_bytes, alloc_size) + alloc_size; + if (mi_inuse_threshold_update(total_inuse_bytes, prof->inuse_interval_size, &prof->inuse_interval_threshold)) { + mi_profiler_snapshot(profiler); + } + } + if (prof->time_interval_secs > 0) { // snapshot whenever `time_interval_secs` seconds have passed + if (mi_time_threshold_update(_mi_clock_now(), prof->time_interval_secs, &prof->time_interval_deadline)) { + mi_profiler_snapshot(profiler); + } + } + return new_threshold; +} + +// Sample free event and update profiling data accordingly +static void mi_cdecl on_free(mi_profiler_t* profiler, mi_profiler_sample_data_t* data, void* ptr, mi_heap_t* heap) +{ + MI_UNUSED(heap); MI_UNUSED(ptr); + mi_pprof_profiler_t* prof = downcast(profiler); + if (data!=NULL) { + mi_assert(data->user_data_size >= 2*sizeof(void*)); + mi_location_t* loc = (mi_location_t*)data->user_data[0]; + const size_t freed_bytes = (size_t)((uintptr_t)data->user_data[1]); + if (loc!=NULL) { + // different threads can free allocations that share the same (allocating-thread, + // callstack) location concurrently, so these updates must be atomic. + mi_atomic_sub_relaxed(&loc->inuse_bytes, freed_bytes); + mi_atomic_decrement_relaxed(&loc->inuse_count); + } + if (prof->inuse_interval_size > 0) { // keep the (global) in-use total in sync while interval-based snapshotting is enabled + mi_atomic_sub_relaxed(&prof->inuse_bytes, freed_bytes); + } + } +} + +static void mi_cdecl on_snapshot(mi_profiler_t* profiler) { + mi_pprof_profiler_t* prof = downcast(profiler); + mi_pprof_profiler_snapshot(prof); +} + +// Create a new profiler. If `base_file_name` ends with a recognized extension (`.heap` or `.text`), +// that extension selects the text snapshot format and is stripped from the stored base file name +// (the actual snapshot files always get their own `..heap`/`..pb` extension, see `mi_pprof_profiler_snapshot`). +// Any other extension (or none) keeps the default (protobuf) snapshot format and is left untouched. +mi_profiler_t* mi_pprof_profiler_new(size_t initial_threshold, const char* base_file_name, size_t alloc_interval_size, size_t inuse_interval_size, size_t time_interval_secs) { + // heap just for the profiler itself + mi_heap_t* heap = mi_heap_new(); + if (heap == NULL) return NULL; + mi_heap_profile_disable(heap); // don't sample allocations in this heap + + // allocate and initialize the profiler structure from this heap + mi_pprof_profiler_t* prof = mi_heap_zalloc_tp(mi_pprof_profiler_t,heap); + if (prof == NULL) { mi_heap_delete(heap); return NULL; } + prof->sample_threshold = initial_threshold; + prof->profile_heap = heap; + prof->base_file_name = (base_file_name != NULL ? mi_heap_strndup(heap, base_file_name, 1024) : NULL); + prof->format_text = false; // default: protobuf format, unless overridden by a recognized extension below + if (prof->base_file_name != NULL) { + // only look for a '.' extension after the last path separator (if any) + const char* start = prof->base_file_name; + const char* sep = _mi_strrchr(start, '/'); + if (sep != NULL) start = sep + 1; + sep = _mi_strrchr(start, '\\'); + if (sep != NULL) start = sep + 1; + const char* dot = _mi_strrchr(start, '.'); + if (dot != NULL) { + if (_mi_streq(dot + 1, "heap") || _mi_streq(dot + 1, "text")) { + prof->format_text = true; + } + prof->base_file_name[dot - prof->base_file_name] = 0; // always strip the extension, whether recognized or not + } + } + prof->alloc_interval_size = (alloc_interval_size > (size_t)MI_SSIZE_MAX ? (size_t)MI_SSIZE_MAX : alloc_interval_size); // cap so it always fits in a `mi_ssize_t` + mi_atomic_storess_relaxed(&prof->alloc_interval_countdown, (mi_ssize_t)prof->alloc_interval_size); // 0 if disabled: `on_alloc` never decrements/checks in that case + prof->inuse_interval_size = inuse_interval_size; + mi_atomic_store_relaxed(&prof->inuse_interval_threshold, prof->inuse_interval_size); // 0 if disabled (`inuse_interval_size` defaults to 0): `on_alloc` never checks in that case + prof->time_interval_secs = time_interval_secs; + prof->start_time_secs = mi_prim_time_secs(); + prof->start_clock = _mi_clock_now(); + mi_atomic_storei64_relaxed(&prof->time_interval_deadline, prof->start_clock + (mi_msecs_t)prof->time_interval_secs * 1000); // deadline stays unreachable (now, i.e. always due) if disabled, but `on_alloc` never checks in that case + if (!mi_locations_init(heap, &prof->locations)) { + // `mi_heap_destroy` frees the live blocks of this private heap, i.e. the + // profiler object and the base file name. (`mi_heap_delete` would instead + // move them to the main heap, losing our only reference.) + mi_heap_destroy(heap); + return NULL; + } + prof->profiler.initial_sample_rate = initial_threshold; + prof->profiler.sample_data_size = 2*sizeof(void*); // we store the location and the allocation size in the sample data + prof->profiler.on_alloc = &on_alloc; + prof->profiler.on_free = &on_free; + prof->profiler.on_snapshot = &on_snapshot; + return &prof->profiler; +} + +void mi_pprof_profiler_delete(mi_profiler_t* profiler) { + if (profiler == NULL) return; + mi_pprof_profiler_t* prof = downcast(profiler); + mi_heap_t* heap = prof->profile_heap; + mi_locations_done(heap, &prof->locations); + mi_free(prof->base_file_name); + mi_free(prof); + mi_heap_delete(heap); +} + +// --------------------------------------------------------------------------- +// A basic hash table of locations. All memory (the bucket array, the +// locations, and their callstack frames) is allocated from the profiler's +// own heap so it does not interfere with the profiled allocations. +// Collisions are resolved with simple chaining. +// +// `on_alloc` and `on_free` (and thus `mi_locations_find_or_insert`) can run +// concurrently on multiple threads and are made thread-safe without a lock: +// a bucket is a singly-linked list that only ever grows by prepending a new, +// fully initialized location; once published, a location's identifying +// fields (`hash`, `thread_id`, `callstack`) never change again. Moreover, +// since the hash (and thus the location itself) is derived from the calling +// thread's own id, a location is only ever looked up by the same thread +// that creates it, so there is never a concurrent insert race for the +// same location. +// --------------------------------------------------------------------------- + +// A reasonably large prime bucket count to keep collision chains short +#define MI_LOCATIONS_BUCKET_COUNT (16 * 1024 - 3) /* 16381, prime */ + +// FNV-1a hash, mixed over the thread id and the (precomputed) callstack hash; +#if SIZE_MAX > UINT32_MAX +#define MI_FNV_OFFSET_BASIS ((size_t)0xcbf29ce484222325ULL) // FNV-1a 64-bit offset basis +#define MI_FNV_PRIME ((size_t)0x100000001b3ULL) // FNV-1a 64-bit prime +#else +#define MI_FNV_OFFSET_BASIS ((size_t)0x811c9dc5UL) // FNV-1a 32-bit offset basis +#define MI_FNV_PRIME ((size_t)0x01000193UL) // FNV-1a 32-bit prime +#endif + +static size_t mi_location_hash(mi_threadid_t thread_id, size_t callstack_hash) { + size_t hash = MI_FNV_OFFSET_BASIS; + const size_t prime = MI_FNV_PRIME; + hash = (hash ^ (size_t)thread_id) * prime; + hash = (hash ^ callstack_hash) * prime; + return hash; +} + +// Initialize a location hash table +static bool mi_locations_init(mi_heap_t* heap, mi_locations_t* locations) { + locations->bucket_count = MI_LOCATIONS_BUCKET_COUNT; + locations->buckets = (_Atomic(mi_location_t*)*)mi_heap_zalloc(heap, locations->bucket_count * sizeof(*locations->buckets)); + mi_atomic_store_relaxed(&locations->count, (size_t)0); + return (locations->buckets != NULL); +} + +// Free all memory associated with the location hash table. +// Not thread-safe: assumes no concurrent `on_alloc`/`on_free` (see the note above). +static void mi_locations_done(mi_heap_t* heap, mi_locations_t* locations) { + MI_UNUSED(heap); + if (locations->buckets == NULL) return; + for (size_t i = 0; i < locations->bucket_count; i++) { + mi_location_t* loc = mi_atomic_load_ptr_relaxed(mi_location_t, &locations->buckets[i]); + while (loc != NULL) { + mi_location_t* next = mi_atomic_load_ptr_relaxed(mi_location_t, &loc->next); + if (loc->callstack.frames != NULL) { + mi_free(loc->callstack.frames); + } + mi_free(loc); + loc = next; + } + } + mi_free(locations->buckets); + locations->buckets = NULL; + mi_atomic_store_relaxed(&locations->count, (size_t)0); +} + +// Match a location's identifying fields; these never change after publication. +static bool mi_location_matches(const mi_location_t* loc, size_t hash, mi_threadid_t thread_id, const mi_callstack_t* callstack) { + return (loc->hash == hash && loc->thread_id == thread_id && + loc->callstack.count == callstack->count && + (callstack->count == 0 || mi_memcmp(loc->callstack.frames, callstack->frames, callstack->count * sizeof(void*)) == 0)); +} + +// Find an existing location matching `thread_id` and `callstack`, or insert +// a fresh, zero-initialized one (with `hash` and identifying fields filled in) +// if none exists yet. Returns NULL only on allocation failure. +// Thread-safe (see the note above): the common case (an existing location) +// is a lock-free, read-only traversal using relaxed loads; only inserting a +// genuinely new location uses a CAS (and even then, only ever races against +// other threads inserting *different* locations into the same bucket). +static mi_location_t* mi_locations_find_or_insert(mi_heap_t* heap, mi_locations_t* locations, mi_threadid_t thread_id, const mi_callstack_t* callstack, size_t callstack_hash) { + if (locations->buckets == NULL) return NULL; + const size_t hash = mi_location_hash(thread_id, callstack_hash); + const size_t idx = hash % locations->bucket_count; + _Atomic(mi_location_t*)* bucket = &locations->buckets[idx]; + + // fast path: lock-free, read-only lookup in the (possibly concurrently growing) chain + for (mi_location_t* loc = mi_atomic_load_ptr_relaxed(mi_location_t, bucket); loc != NULL; + loc = mi_atomic_load_ptr_relaxed(mi_location_t, &loc->next)) + { + if (mi_location_matches(loc, hash, thread_id, callstack)) return loc; + } + + // not found: allocate and fully initialize a new location; it is not yet + // visible to any other thread so plain (non-atomic) stores are fine here. + mi_location_t* loc = mi_heap_zalloc_tp(mi_location_t,heap); + if (loc == NULL) return NULL; + if (callstack->count > 0) { + loc->callstack.frames = (void**)mi_heap_mallocn(heap, callstack->count, sizeof(void*)); + if (loc->callstack.frames == NULL) { mi_free(loc); return NULL; } // free the not-yet-published location + mi_memcpy(loc->callstack.frames, callstack->frames, callstack->count * sizeof(void*)); + } + loc->callstack.count = callstack->count; + loc->thread_id = thread_id; + loc->hash = hash; + + // publish the new location at the head of the bucket with a CAS loop. Since + // a location's key includes the calling thread's own id, no other thread + // can ever race to insert this *same* key -- a CAS can only fail because + // some other thread prepended an unrelated location to this bucket in the + // meantime, so we simply retry with the updated head (no duplicate check + // needed, unlike a typical lock-free insert). + mi_location_t* head = mi_atomic_load_ptr_relaxed(mi_location_t, bucket); + do { + mi_atomic_store_ptr_relaxed(mi_location_t, &loc->next, head); // `loc` not yet published: plain store is fine + } while (!mi_atomic_cas_ptr_weak_release(mi_location_t, bucket, &head, loc)); // `head` is updated to the current value on failure + mi_atomic_increment_relaxed(&locations->count); // just a statistic + return loc; +} + + + +// --------------------------------------------------------------------------- +// Avoid OS specific code when MI_PROFILE_USE_BUILTIN is not enabled +// --------------------------------------------------------------------------- + +#if !MI_PROFILE_USE_BUILTIN + +static size_t mi_prim_backtrace(void** buffer, size_t max_depth, size_t* hash) { + MI_UNUSED(buffer); + MI_UNUSED(max_depth); + *hash = 0; + return 0; +} + +static int64_t mi_prim_time_secs(void) { + return 0; +} + +void mi_pprof_profiler_snapshot(mi_pprof_profiler_t* profiler) { + MI_UNUSED(profiler); +} + +#else + +#include // FILE, fopen, fprintf, fclose +#include // time +#if defined(_WIN32) +#include // _O_CREAT, _O_TRUNC, _O_WRONLY, _O_BINARY, _O_TEXT +#include // _sopen_s, _fdopen, _close +#include // _SH_DENYNO +#include // _S_IREAD, _S_IWRITE +#elif MI_HAS_UNISTDH +#include // open, O_CREAT, O_EXCL, O_NOFOLLOW +#include // close, getuid, geteuid, getgid, getegid, issetugid +#include // S_IRUSR, S_IWUSR +#endif + +// forward declarations; implemented further below. +static void mi_pprof_write_mapped_libraries(FILE* f); +static bool mi_pprof_snapshot_text(mi_pprof_profiler_t* prof, const char* fname); +static bool mi_pprof_snapshot_proto(mi_pprof_profiler_t* prof, const char* fname); + + +// Detect if we are running with elevated privileges (e.g. a setuid/setgid executable or similar), +// in which case we must not trust `MIMALLOC_PROFILE` environment. +#if defined(_WIN32) +static bool mi_is_elevated_process(void) { + return false; // no direct setuid/setgid equivalent; a UAC elevation does not inherit the parent's environment +} +#elif MI_HAS_UNISTDH && (defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) || defined(__DragonFly__)) +static bool mi_is_elevated_process(void) { + return (issetugid() != 0); +} +#elif MI_HAS_UNISTDH +static bool mi_is_elevated_process(void) { + return (getuid() != geteuid() || getgid() != getegid()); +} +#else +static bool mi_is_elevated_process(void) { + return false; // no portable way to check without `unistd.h`; assume not elevated +} +#endif + + +static int64_t mi_prim_time_secs(void) { + const time_t t = time(NULL); + return (t == (time_t)-1 ? 0 : (int64_t)t); +} + + +// Open a snapshot file for writing, refusing to follow a symlink at `fname` (to avoid an +// attacker redirecting the write to an arbitrary target) and restricting the file's +// permissions to the owner only (snapshots can contain sensitive address/callstack data). +// On platforms without these POSIX/Win32 facilities (e.g. wasm/Emscripten), we fall +// back to a fully portable plain `fopen` (without the symlink/permission hardening). +static FILE* mi_pprof_fopen(const char* fname, bool binary) { + #if defined(_WIN32) + int oflag = _O_CREAT | _O_TRUNC | _O_WRONLY | (binary ? _O_BINARY : _O_TEXT); + int fd = -1; + if (_sopen_s(&fd, fname, oflag, _SH_DENYNO, _S_IREAD | _S_IWRITE) != 0 || fd < 0) return NULL; + FILE* f = _fdopen(fd, binary ? "wb" : "w"); + if (f == NULL) { _close(fd); } + return f; + #elif MI_HAS_UNISTDH + int oflag = O_CREAT | O_TRUNC | O_WRONLY; + #if defined(O_NOFOLLOW) + oflag |= O_NOFOLLOW; // fail (ELOOP) if `fname` is a symlink + #endif + const int fd = open(fname, oflag, S_IRUSR | S_IWUSR); // restrict to owner read/write only + if (fd < 0) return NULL; + FILE* f = fdopen(fd, binary ? "wb" : "w"); + if (f == NULL) { close(fd); } + return f; + #else + return fopen(fname, binary ? "wb" : "w"); // fully portable fallback: no symlink/permission hardening available + #endif +} + +// Write out the current profiler data to a new snapshot file named +// `..` with an incrementing sequence number +// (`.heap` for the text format, mimicking the naming used by the original +// (gperftools) pprof heap profiler; `.pb` for the (uncompressed) protobuf +// format). The format is chosen when the profiler is created (see +// `mi_pprof_profiler_new`). The base file name is also set at creation time; +// if it is NULL, no snapshot is written and this function does nothing. +static void mi_pprof_profiler_snapshot(mi_pprof_profiler_t* prof) { + if (prof == NULL) return; + bool was_running = mi_profiler_stop(&prof->profiler); + mi_locations_t* locations = &prof->locations; + if (locations->buckets != NULL && prof->base_file_name != NULL) { + const size_t seq = ++prof->snapshot_count; + char fname[1024]; + bool ok; + if (!prof->format_text) { + _mi_snprintf(fname, sizeof(fname), "%s.%04zu.pb", prof->base_file_name, seq); + ok = mi_pprof_snapshot_proto(prof, fname); + } + else { + _mi_snprintf(fname, sizeof(fname), "%s.%04zu.heap", prof->base_file_name, seq); + ok = mi_pprof_snapshot_text(prof, fname); + } + // A snapshot is best effort: it can fail when the profiler's own heap is out + // of memory (or the file cannot be created), in which case no (or only a + // partial) snapshot is written. Never let that affect the profiled program. + if (ok) { + _mi_verbose_message("pprof profiler snapshot: %s\n", fname); + } + else { + _mi_verbose_message("pprof profiler: unable to write snapshot (out of memory?)\n"); + } + } + if (was_running) { mi_profiler_start(&prof->profiler); } +} + +// --------------------------------------------------------------------------- +// Basic backtraces +// --------------------------------------------------------------------------- + +#if MI_HAS_EXECINFOH || MI_HAS_LIBUNWINDH +static size_t mi_backtrace_hash(void** frames, size_t depth) { + size_t hash = MI_FNV_OFFSET_BASIS; + for (size_t i = 0; i < depth; i++) { + hash = (hash ^ (size_t)((uintptr_t)frames[i])) * MI_FNV_PRIME; + } + return hash; +} +#endif + +#if _WIN32 +#include // CaptureStackBackTrace +static mi_decl_noinline size_t mi_prim_backtrace(void** buffer, size_t max_depth, size_t* hash) { + if (max_depth > ULONG_MAX) { max_depth = ULONG_MAX; } + // On Windows XP/Server 2003, `FramesToSkip + FramesToCapture` must be less than 63; + // clamp defensively so we never exceed this, regardless of the actual OS version. + const size_t skip = 1 + MI_BACKTRACE_SKIP_FRAMES; // +1 for `mi_prim_backtrace`'s own frame + const size_t win_limit = 62; // strictly less than 63 + if (skip + max_depth >= win_limit) { max_depth = (skip <= win_limit ? win_limit - skip : 0); } + if (max_depth == 0) { *hash = 0; return 0; } + DWORD win_hash = 0; + const size_t depth = (size_t)CaptureStackBackTrace((ULONG)skip, (ULONG)max_depth, buffer, &win_hash); + *hash = (size_t)win_hash; // use the hash computed by `CaptureStackBackTrace` itself + return depth; +} +#elif MI_HAS_EXECINFOH +#include // backtrace +static mi_decl_noinline size_t mi_prim_backtrace(void** buffer, size_t max_depth, size_t* hash) { + const size_t skip = 1 + MI_BACKTRACE_SKIP_FRAMES; // +1 for `mi_prim_backtrace`'s own frame + if (max_depth <= skip) { *hash = 0; return 0; } + if (max_depth > INT_MAX) { max_depth = INT_MAX; } + const size_t raw_depth = (size_t)backtrace(buffer, (int)(max_depth - skip)); + if (raw_depth <= skip) { *hash = 0; return 0; } + const size_t depth = raw_depth - skip; + memmove(buffer, buffer + skip, depth * sizeof(void*)); + *hash = mi_backtrace_hash(buffer, depth); // no platform-provided hash, compute our own + return depth; +} +#elif MI_HAS_LIBUNWINDH +#define UNW_LOCAL_ONLY +#include // unw_backtrace +static mi_decl_noinline size_t mi_prim_backtrace(void** buffer, size_t max_depth, size_t* hash) { + // used as a fallback on platforms without `execinfo.h` (e.g. musl-based Linux like Alpine) + const size_t skip = 1 + MI_BACKTRACE_SKIP_FRAMES; // +1 for `mi_prim_backtrace`'s own frame + if (max_depth <= skip) { *hash = 0; return 0; } + if (max_depth > INT_MAX) { max_depth = INT_MAX; } + const size_t raw_depth = (size_t)unw_backtrace(buffer, (int)(max_depth - skip)); + if (raw_depth <= skip) { *hash = 0; return 0; } + const size_t depth = raw_depth - skip; + memmove(buffer, buffer + skip, depth * sizeof(void*)); + *hash = mi_backtrace_hash(buffer, depth); // no platform-provided hash, compute our own + return depth; +} +#else +static size_t mi_prim_backtrace(void** buffer, size_t max_depth, size_t* hash) { + MI_UNUSED(buffer); + MI_UNUSED(max_depth); + *hash = 0; + return 0; +} +#endif + + +// --------------------------------------------------------------------------- +// Write out mapped libraries +// --------------------------------------------------------------------------- + +// A loaded module's address range and (if known) its file path; used both to +// print the `MAPPED_LIBRARIES` trailer of the text format and to build the +// `Mapping` entries of the protobuf format (see `mi_pprof_snapshot_proto`). +typedef struct { + uintptr_t start; + uintptr_t end; + char* path; // heap-allocated (owned), or NULL if unknown +} mi_pprof_module_t; + +typedef struct { + mi_pprof_module_t* modules; + size_t count; + size_t capacity; + bool failed; // set if an allocation failed; the mapping list is then incomplete +} mi_pprof_modules_t; + +static void mi_pprof_modules_add(mi_heap_t* heap, mi_pprof_modules_t* mods, uintptr_t start, uintptr_t end, const char* path) { + if (mods->failed) return; + if (mods->count == mods->capacity) { + const size_t new_capacity = (mods->capacity == 0 ? 16 : mods->capacity * 2); + mi_pprof_module_t* const new_modules = (mi_pprof_module_t*)mi_heap_realloc(heap, mods->modules, new_capacity * sizeof(*mods->modules)); + if (new_modules == NULL) { mods->failed = true; return; } + mods->modules = new_modules; + mods->capacity = new_capacity; + } + char* const copy = (path != NULL ? mi_heap_strndup(heap, path, 1024) : NULL); + if (path != NULL && copy == NULL) { mods->failed = true; return; } + mi_pprof_module_t* m = &mods->modules[mods->count++]; + m->start = start; + m->end = end; + m->path = copy; +} + +static void mi_pprof_modules_done(mi_pprof_modules_t* mods) { + for (size_t i = 0; i < mods->count; i++) { mi_free(mods->modules[i].path); } + mi_free(mods->modules); + mods->modules = NULL; + mods->count = 0; + mods->capacity = 0; +} + +// On Linux, `pprof` can symbolize addresses using the `MAPPED_LIBRARIES` trailer +// which is just a copy of `/proc/self/maps`. On macOS there is no equivalent +// file so we reconstruct a similarly formatted line (`start-end perm ... path`) +// per loaded Mach-O image using the dyld APIs: the range is taken from the +// image's `__TEXT` segment, which is also the "load address" that tools like +// `atos` use to symbolize addresses (`atos -o -l `). On +// Windows we walk the process address space with `VirtualQuery` +// On other platforms we leave this section empty (`pprof` can still work +// without it if not using PIE, or when combined with tools like `atos` / +// `addr2line` / a symbol server). + +// A small wrapper that formats with our own minimal `_mi_vsnprintf` (so call +// sites can use its `%l8x`/`%l8d`/`%tx` extensions instead of 's +// `PRIx64`/`PRIu64`/`PRIxPTR` macros) and writes the result directly to a `FILE*`. +// (Distinct from the existing `_mi_fprintf(mi_output_fun*, void*, ...)` in +// `util/options.c`, which targets mimalloc's redirectable output callback instead +// of an arbitrary `FILE*`.) +static void mi_fprintf(FILE* f, const char* fmt, ...) { + char buf[512]; + va_list args; + va_start(args, fmt); + _mi_vsnprintf(buf, sizeof(buf), fmt, args); + va_end(args); + fputs(buf, f); +} + +#if defined(_WIN32) +#include + +// Walk the process address space, calling `on_module(ctx, base, end, path)` +// once for each loaded module. Consecutive `MEM_IMAGE` regions that belong to +// the same module (same allocation base) are merged into a single range. +static void mi_walk_modules_win32(void (*on_module)(void* ctx, uintptr_t base, uintptr_t end, const char* path), void* ctx) { + SYSTEM_INFO si; + GetSystemInfo(&si); + uint8_t* addr = (uint8_t*)si.lpMinimumApplicationAddress; + uint8_t* const addr_max = (uint8_t*)si.lpMaximumApplicationAddress; + void* cur_base = NULL; + uintptr_t cur_end = 0; + while (addr < addr_max) { + MEMORY_BASIC_INFORMATION mbi; + const SIZE_T n = VirtualQuery(addr, &mbi, sizeof(mbi)); + if (n == 0) break; // no more (queryable) regions + if (mbi.Type == MEM_IMAGE && mbi.State != MEM_FREE) { + if (mbi.AllocationBase == cur_base) { + // extend the current module's range + cur_end = (uintptr_t)mbi.BaseAddress + mbi.RegionSize; + } + else { + // a new module starts: flush the previous one first + if (cur_base != NULL) { + char path[MAX_PATH]; + const DWORD len = GetModuleFileNameA((HMODULE)cur_base, path, (DWORD)sizeof(path)); + on_module(ctx, (uintptr_t)cur_base, cur_end, (len > 0 ? path : "")); + } + cur_base = mbi.AllocationBase; + cur_end = (uintptr_t)mbi.BaseAddress + mbi.RegionSize; + } + } + else if (cur_base != NULL) { + // the module's regions ended: flush it + char path[MAX_PATH]; + const DWORD len = GetModuleFileNameA((HMODULE)cur_base, path, (DWORD)sizeof(path)); + on_module(ctx, (uintptr_t)cur_base, cur_end, (len > 0 ? path : "")); + cur_base = NULL; + } + const uint8_t* next = (const uint8_t*)mbi.BaseAddress + mbi.RegionSize; + if (next <= addr) break; // guard against a non-advancing region + addr = (uint8_t*)next; + } + if (cur_base != NULL) { + char path[MAX_PATH]; + const DWORD len = GetModuleFileNameA((HMODULE)cur_base, path, (DWORD)sizeof(path)); + on_module(ctx, (uintptr_t)cur_base, cur_end, (len > 0 ? path : "")); + } +} + +static void mi_write_mapped_libraries_on_module_win32(void* ctx, uintptr_t base, uintptr_t end, const char* path) { + FILE* f = (FILE*)ctx; + mi_fprintf(f, "%tx-%tx r-xp 00000000 00:00 0 %s\n", base, end, path); +} + +static void mi_write_mapped_libraries_win32(FILE* f) { + mi_walk_modules_win32(&mi_write_mapped_libraries_on_module_win32, f); +} + +typedef struct { mi_heap_t* heap; mi_pprof_modules_t* mods; } mi_pprof_collect_modules_win32_ctx_t; + +static void mi_pprof_collect_modules_win32_on_module(void* ctx, uintptr_t base, uintptr_t end, const char* path) { + mi_pprof_collect_modules_win32_ctx_t* c = (mi_pprof_collect_modules_win32_ctx_t*)ctx; + mi_pprof_modules_add(c->heap, c->mods, base, end, (path[0] != 0 ? path : NULL)); +} + +static void mi_pprof_collect_modules_win32(mi_heap_t* heap, mi_pprof_modules_t* mods) { + mi_pprof_collect_modules_win32_ctx_t ctx = { heap, mods }; + mi_walk_modules_win32(&mi_pprof_collect_modules_win32_on_module, &ctx); +} + +#elif defined(__APPLE__) +#include +#include + +// Find the (slid) virtual address range of the `__TEXT` segment of a loaded +// Mach-O image; this is the image's load address as used for symbolication. +static bool mi_macho_image_range(const struct mach_header* mh, intptr_t slide, uintptr_t* start, uintptr_t* end) { + const bool is64 = (mh->magic == MH_MAGIC_64 || mh->magic == MH_CIGAM_64); + const uint8_t* cmd_ptr = (const uint8_t*)mh + (is64 ? sizeof(struct mach_header_64) : sizeof(struct mach_header)); + for (uint32_t c = 0; c < mh->ncmds; c++) { + const struct load_command* lc = (const struct load_command*)cmd_ptr; + if (is64 && lc->cmd == LC_SEGMENT_64) { + const struct segment_command_64* seg = (const struct segment_command_64*)lc; + if (_mi_streq(seg->segname, SEG_TEXT)) { + *start = (uintptr_t)seg->vmaddr + (uintptr_t)slide; + *end = *start + (uintptr_t)seg->vmsize; + return true; + } + } + else if (!is64 && lc->cmd == LC_SEGMENT) { + const struct segment_command* seg = (const struct segment_command*)lc; + if (_mi_streq(seg->segname, SEG_TEXT)) { + *start = (uintptr_t)seg->vmaddr + (uintptr_t)slide; + *end = *start + (uintptr_t)seg->vmsize; + return true; + } + } + cmd_ptr += lc->cmdsize; + } + return false; // no __TEXT segment found +} + +static void mi_write_mapped_libraries_macos(FILE* f) { + const uint32_t count = _dyld_image_count(); + for (uint32_t i = 0; i < count; i++) { + const struct mach_header* mh = _dyld_get_image_header(i); + const char* name = _dyld_get_image_name(i); + if (mh == NULL || name == NULL) continue; + const intptr_t slide = _dyld_get_image_vmaddr_slide(i); + uintptr_t start = 0; + uintptr_t end = 0; + if (!mi_macho_image_range(mh, slide, &start, &end)) continue; + mi_fprintf(f, "%tx-%tx r-xp 00000000 00:00 0 %s\n", start, end, name); + } +} + +static void mi_pprof_collect_modules_macos(mi_heap_t* heap, mi_pprof_modules_t* mods) { + const uint32_t count = _dyld_image_count(); + for (uint32_t i = 0; i < count; i++) { + const struct mach_header* mh = _dyld_get_image_header(i); + const char* name = _dyld_get_image_name(i); + if (mh == NULL || name == NULL) continue; + const intptr_t slide = _dyld_get_image_vmaddr_slide(i); + uintptr_t start = 0; + uintptr_t end = 0; + if (!mi_macho_image_range(mh, slide, &start, &end)) continue; + mi_pprof_modules_add(heap, mods, start, end, name); + } +} + +#elif defined(__linux__) +#include // strtoull + +// Copy `/proc/self/maps` verbatim; this is exactly the format `pprof` expects. +static void mi_write_mapped_libraries_linux(FILE* f) { + FILE* maps = fopen("/proc/self/maps", "r"); + if (maps != NULL) { + char line[512]; + while (fgets(line, sizeof(line), maps) != NULL) { + fputs(line, f); + } + fclose(maps); + } +} + +// Parse `/proc/self/maps` lines of the form +// `- ` (path may be absent +// for anonymous mappings, which are skipped here since they are not useful +// for symbolization). +static void mi_pprof_collect_modules_linux(mi_heap_t* heap, mi_pprof_modules_t* mods) { + FILE* maps = fopen("/proc/self/maps", "r"); + if (maps == NULL) return; + char line[512]; + while (fgets(line, sizeof(line), maps) != NULL) { + char* end_ptr = NULL; + const uintptr_t start = (uintptr_t)strtoull(line, &end_ptr, 16); + if (end_ptr == NULL || *end_ptr != '-') continue; + const uintptr_t end = (uintptr_t)strtoull(end_ptr + 1, &end_ptr, 16); + // the path (if present) is the last whitespace-separated field on the line + const char* path = strrchr(line, ' '); + if (path == NULL) continue; + while (*path == ' ') path++; + size_t len = strlen(path); + while (len > 0 && (path[len-1] == '\n' || path[len-1] == '\r')) { len--; } + if (len == 0 || path[0] == '[') continue; // skip anonymous mappings like `[heap]`, `[stack]` + char pathbuf[512]; + if (len >= sizeof(pathbuf)) { len = sizeof(pathbuf) - 1; } + memcpy(pathbuf, path, len); + pathbuf[len] = 0; + mi_pprof_modules_add(heap, mods, start, end, pathbuf); + } + fclose(maps); +} + +#elif defined(__FreeBSD__) || defined(__DragonFly__) +#include // struct kinfo_vmentry +#include // kinfo_getvmmap +#include // getpid +#include // free + +// FreeBSD (and DragonFly, which forked from FreeBSD) do not reliably provide Linux's +// `/proc/self/maps` (procfs is optional and rarely mounted by default); use the native +// `kinfo_getvmmap` API instead (declared in , requires linking `-lutil`) to +// enumerate the process's virtual memory mappings. +static void mi_write_mapped_libraries_bsd(FILE* f) { + int cnt = 0; + struct kinfo_vmentry* vmmap = kinfo_getvmmap(getpid(), &cnt); + if (vmmap == NULL) return; + for (int i = 0; i < cnt; i++) { + const struct kinfo_vmentry* kve = &vmmap[i]; + if (kve->kve_path[0] == 0) continue; // skip anonymous mappings + mi_fprintf(f, "%l8x-%l8x r-xp 00000000 00:00 0 %s\n", (uint64_t)kve->kve_start, (uint64_t)kve->kve_end, kve->kve_path); + } + free(vmmap); +} + +static void mi_pprof_collect_modules_bsd(mi_heap_t* heap, mi_pprof_modules_t* mods) { + int cnt = 0; + struct kinfo_vmentry* vmmap = kinfo_getvmmap(getpid(), &cnt); + if (vmmap == NULL) return; + for (int i = 0; i < cnt; i++) { + const struct kinfo_vmentry* kve = &vmmap[i]; + if (kve->kve_path[0] == 0) continue; // skip anonymous mappings + mi_pprof_modules_add(heap, mods, (uintptr_t)kve->kve_start, (uintptr_t)kve->kve_end, kve->kve_path); + } + free(vmmap); +} +#endif + +static void mi_pprof_collect_modules(mi_heap_t* heap, mi_pprof_modules_t* mods) { + mods->modules = NULL; + mods->count = 0; + mods->capacity = 0; + mods->failed = false; + #if defined(__linux__) + mi_pprof_collect_modules_linux(heap, mods); + #elif defined(_WIN32) + mi_pprof_collect_modules_win32(heap, mods); + #elif defined(__APPLE__) + mi_pprof_collect_modules_macos(heap, mods); + #elif defined(__FreeBSD__) || defined(__DragonFly__) + mi_pprof_collect_modules_bsd(heap, mods); + #else + MI_UNUSED(heap); + #endif +} + +static void mi_pprof_write_mapped_libraries(FILE* f) { + fprintf(f, "MAPPED_LIBRARIES:\n"); + #if defined(__linux__) + mi_write_mapped_libraries_linux(f); + #elif defined(_WIN32) + mi_write_mapped_libraries_win32(f); + #elif defined(__APPLE__) + mi_write_mapped_libraries_macos(f); + #elif defined(__FreeBSD__) || defined(__DragonFly__) + mi_write_mapped_libraries_bsd(f); + #endif +} + + +// --------------------------------------------------------------------------- +// Writing the profile in the (original, textual) pprof heap profile format +// --------------------------------------------------------------------------- + +// Write out the current profiler data in the pprof heap profile text format: +// a header line with the totals, followed by one line per unique call +// location, and a trailer with the mapped libraries (used by `pprof` to +// symbolize the addresses). +static bool mi_pprof_snapshot_text(mi_pprof_profiler_t* prof, const char* fname) { + mi_locations_t* locations = &prof->locations; + FILE* f = mi_pprof_fopen(fname, false); + if (f == NULL) return false; + + // first pass: compute the totals over all locations + // (snapshot is assumed to run single-threaded while sampling is stopped, so + // `alloc_count`/`alloc_bytes` can be read directly; `inuse_count`/`inuse_bytes` + // are still `_Atomic` fields so we use a relaxed load for those) + uint64_t total_inuse_objects = 0; + uint64_t total_inuse_bytes = 0; + uint64_t total_alloc_objects = 0; + uint64_t total_alloc_bytes = 0; + for (size_t i = 0; i < locations->bucket_count; i++) { + for (mi_location_t* loc = mi_atomic_load_ptr_relaxed(mi_location_t, &locations->buckets[i]); loc != NULL; loc = mi_atomic_load_ptr_relaxed(mi_location_t, &loc->next)) { + total_inuse_objects += (uint64_t)mi_atomic_load_relaxed(&loc->inuse_count); + total_inuse_bytes += (uint64_t)mi_atomic_load_relaxed(&loc->inuse_bytes); + total_alloc_objects += (uint64_t)loc->alloc_count; + total_alloc_bytes += (uint64_t)loc->alloc_bytes; + } + } + + mi_fprintf(f, "heap profile: %6l8d: %8l8d [%6l8d: %8l8d] @ heapprofile\n", + total_inuse_objects, total_inuse_bytes, total_alloc_objects, total_alloc_bytes); + + // second pass: write one line per unique call location + for (size_t i = 0; i < locations->bucket_count; i++) { + for (mi_location_t* loc = mi_atomic_load_ptr_relaxed(mi_location_t, &locations->buckets[i]); loc != NULL; loc = mi_atomic_load_ptr_relaxed(mi_location_t, &loc->next)) { + const size_t inuse_count = mi_atomic_load_relaxed(&loc->inuse_count); + const size_t inuse_bytes = mi_atomic_load_relaxed(&loc->inuse_bytes); + mi_fprintf(f, "%6l8d: %8l8d [%6l8d: %8l8d] @", + (uint64_t)inuse_count, (uint64_t)inuse_bytes, + (uint64_t)loc->alloc_count, (uint64_t)loc->alloc_bytes); + for (size_t j = 0; j < loc->callstack.count; j++) { + mi_fprintf(f, " 0x%tx", (uintptr_t)loc->callstack.frames[j]); + } + fprintf(f, "\n"); + } + } + + mi_pprof_write_mapped_libraries(f); + const bool ok = (ferror(f) == 0); + if (fclose(f) != 0) return false; + return ok; +} + + +// --------------------------------------------------------------------------- +// Writing the profile in the modern `pprof` protobuf format +// (`perftools.profiles.Profile`, see +// https://github.com/google/pprof/blob/main/proto/profile.proto) +// +// On-disk, `pprof` profiles are normally gzip-compressed, but `pprof` also +// accepts a *raw* (uncompressed) serialized `Profile` message: it only tries +// to gunzip the input if it starts with the gzip magic bytes, and otherwise +// parses it directly as a protobuf message. We take advantage of that so we +// don't need to implement a deflate/gzip writer. +// +// We still need the locations hash table for this format: samples are +// aggregated per call site (thread + callstack), exactly as for the text +// format, rather than emitting one `Sample` per individual allocation. What +// changes is the *representation* of a callstack: instead of a flat list of +// hex addresses embedded directly in a text line, each frame address becomes +// its own (deduplicated) `Location` entry, referenced by id from `Sample. +// location_id`. The originating thread id -- which the text format doesn't +// record explicitly (it is only implicit in which addresses got sampled) -- +// is attached to each `Sample` via a numeric `Label` (pprof has no dedicated +// thread-id field, but labels are the generic mechanism for this). +// --------------------------------------------------------------------------- + +// A simple growable byte buffer, allocated from the profiler's own heap. +// If an allocation fails, `failed` is set and all further writes become no-ops; +// the caller must check it and discard the whole snapshot. +typedef struct { + mi_heap_t* heap; + uint8_t* data; + size_t size; + size_t capacity; + bool failed; +} mi_pb_buf_t; + +static void mi_pb_init(mi_pb_buf_t* b, mi_heap_t* heap) { + b->heap = heap; + b->data = NULL; + b->size = 0; + b->capacity = 0; + b->failed = false; +} + +static void mi_pb_done(mi_pb_buf_t* b) { + mi_free(b->data); + b->data = NULL; + b->size = 0; + b->capacity = 0; + b->failed = false; +} + +static bool mi_pb_reserve(mi_pb_buf_t* b, size_t extra) { + if (b->failed) return false; + if (b->size + extra <= b->capacity) return true; + size_t new_capacity = (b->capacity == 0 ? 4096 : b->capacity * 2); + while (new_capacity < b->size + extra) { new_capacity *= 2; } + uint8_t* const new_data = (uint8_t*)mi_heap_realloc(b->heap, b->data, new_capacity); + if (new_data == NULL) { b->failed = true; return false; } // keep the old buffer (and capacity) intact + b->data = new_data; + b->capacity = new_capacity; + return true; +} + +static void mi_pb_bytes(mi_pb_buf_t* b, const void* p, size_t n) { + if (n == 0) return; // avoid calling memcpy with a NULL source + if (!mi_pb_reserve(b, n)) return; + memcpy(b->data + b->size, p, n); + b->size += n; +} + +static void mi_pb_byte(mi_pb_buf_t* b, uint8_t byte) { + mi_pb_bytes(b, &byte, 1); +} + +// Unsigned LEB128 (protobuf "varint") encoding. +static void mi_pb_varint(mi_pb_buf_t* b, uint64_t v) { + do { + uint8_t byte = (uint8_t)(v & 0x7F); + v >>= 7; + if (v != 0) { byte |= 0x80; } + mi_pb_byte(b, byte); + } while (v != 0); +} + +static void mi_pb_tag(mi_pb_buf_t* b, uint32_t field, uint32_t wiretype) { + mi_pb_varint(b, ((uint64_t)field << 3) | wiretype); +} + +// Scalar varint fields; proto3 encoders may omit fields set to their default +// (0/false) value, which we do here to keep the output compact. +static void mi_pb_uint64_field(mi_pb_buf_t* b, uint32_t field, uint64_t v) { + if (v == 0) return; + mi_pb_tag(b, field, 0); + mi_pb_varint(b, v); +} + +static void mi_pb_int64_field(mi_pb_buf_t* b, uint32_t field, int64_t v) { + if (v == 0) return; + mi_pb_tag(b, field, 0); + mi_pb_varint(b, (uint64_t)v); // proto3 plain int64 is encoded as a varint of its 2's complement bit pattern +} + +// Length-delimited fields (string/bytes/embedded message) are always emitted, +// even when empty: this matters for `string_table[0]` (which must be the +// empty string) and for repeated message fields (an empty message is still a +// valid, distinct list entry). +static void mi_pb_len_field(mi_pb_buf_t* b, uint32_t field, const void* data, size_t len) { + mi_pb_tag(b, field, 2); + mi_pb_varint(b, len); + mi_pb_bytes(b, data, len); +} + +static void mi_pb_message_field(mi_pb_buf_t* b, uint32_t field, const mi_pb_buf_t* msg) { + if (msg->failed) { b->failed = true; return; } // propagate a failure from the nested message + mi_pb_len_field(b, field, msg->data, msg->size); +} + +// A packed repeated scalar field (used for `Sample.location_id`/`Sample.value`): +// all elements are varint-encoded and concatenated into one length-delimited blob. +static void mi_pb_packed_uint64_field(mi_pb_buf_t* b, uint32_t field, const uint64_t* values, size_t count) { + if (count == 0) return; + mi_pb_buf_t tmp; + mi_pb_init(&tmp, b->heap); + for (size_t i = 0; i < count; i++) { mi_pb_varint(&tmp, values[i]); } + mi_pb_message_field(b, field, &tmp); + mi_pb_done(&tmp); +} + +static void mi_pb_packed_int64_field(mi_pb_buf_t* b, uint32_t field, const int64_t* values, size_t count) { + mi_pb_packed_uint64_field(b, field, (const uint64_t*)values, count); // same varint bit pattern +} + +// --------------------------------------------------------------------------- +// String table: dedups strings into `Profile.string_table` (field 6) and +// hands back the (1-based; index 0 is reserved for "") index of each string. +// A linear search is fine here: a profile typically only has a handful to a +// few hundred distinct strings (sample type names, mapped file paths). +// --------------------------------------------------------------------------- + +typedef struct { + char* str; + int64_t index; +} mi_pb_string_entry_t; + +typedef struct { + mi_heap_t* heap; + mi_pb_string_entry_t* entries; + size_t count; + size_t capacity; + bool failed; // set if an allocation failed; the table is then incomplete + mi_pb_buf_t table; // the serialized `string_table` entries (Profile field 6) +} mi_pb_strings_t; + +static void mi_pb_strings_init(mi_pb_strings_t* strings, mi_heap_t* heap) { + strings->heap = heap; + strings->entries = NULL; + strings->count = 0; + strings->capacity = 0; + strings->failed = false; + mi_pb_init(&strings->table, heap); + mi_pb_len_field(&strings->table, 6, "", 0); // index 0 must always be "" +} + +static void mi_pb_strings_done(mi_pb_strings_t* strings) { + for (size_t i = 0; i < strings->count; i++) { mi_free(strings->entries[i].str); } + mi_free(strings->entries); + mi_pb_done(&strings->table); +} + +static int64_t mi_pb_strings_intern(mi_pb_strings_t* strings, const char* s) { + if (strings->failed) return 0; + if (s == NULL) { s = ""; } + for (size_t i = 0; i < strings->count; i++) { + if (_mi_streq(strings->entries[i].str, s)) { return strings->entries[i].index; } + } + if (strings->count == strings->capacity) { + const size_t new_capacity = (strings->capacity == 0 ? 32 : strings->capacity * 2); + mi_pb_string_entry_t* const new_entries = (mi_pb_string_entry_t*)mi_heap_realloc(strings->heap, strings->entries, new_capacity * sizeof(*strings->entries)); + if (new_entries == NULL) { strings->failed = true; return 0; } + strings->entries = new_entries; + strings->capacity = new_capacity; + } + char* const copy = mi_heap_strndup(strings->heap, s, 1024); + if (copy == NULL) { strings->failed = true; return 0; } + const int64_t index = (int64_t)(strings->count + 1); // index 0 is reserved for "" + strings->entries[strings->count].str = copy; + strings->entries[strings->count].index = index; + strings->count++; + mi_pb_len_field(&strings->table, 6, s, strlen(s)); + if (strings->table.failed) { strings->failed = true; return 0; } + return index; +} + +// --------------------------------------------------------------------------- +// A small single-threaded hash map from a raw frame address to the id of its +// (deduplicated) `Location` entry. Only ever built and consulted from within +// `mi_pprof_snapshot_proto`, so (unlike the locations hash table) it needs no +// atomics at all. +// --------------------------------------------------------------------------- + +typedef struct mi_pb_addr_entry_s { + struct mi_pb_addr_entry_s* next; + uintptr_t addr; + uint64_t id; +} mi_pb_addr_entry_t; + +typedef struct { + mi_heap_t* heap; + size_t bucket_count; + mi_pb_addr_entry_t** buckets; + bool failed; // set if an allocation failed (the map is then unusable) +} mi_pb_addr_map_t; + +static void mi_pb_addr_map_init(mi_pb_addr_map_t* map, mi_heap_t* heap, size_t bucket_count) { + map->heap = heap; + map->bucket_count = (bucket_count == 0 ? 1 : bucket_count); + map->buckets = (mi_pb_addr_entry_t**)mi_heap_zalloc(heap, map->bucket_count * sizeof(*map->buckets)); + map->failed = (map->buckets == NULL); +} + +static void mi_pb_addr_map_done(mi_pb_addr_map_t* map) { + if (map->buckets == NULL) return; + for (size_t i = 0; i < map->bucket_count; i++) { + mi_pb_addr_entry_t* e = map->buckets[i]; + while (e != NULL) { + mi_pb_addr_entry_t* const next = e->next; + mi_free(e); + e = next; + } + } + mi_free(map->buckets); + map->buckets = NULL; +} + +// Returns the existing location id for `addr`, or assigns it `next_id` and +// returns that (setting `*is_new` to true) if this is the first time `addr` +// is seen. +static uint64_t mi_pb_addr_map_find_or_insert(mi_pb_addr_map_t* map, uintptr_t addr, uint64_t next_id, bool* is_new) { + if (map->failed || map->buckets == NULL) { map->failed = true; *is_new = false; return 0; } + const size_t idx = (size_t)(addr ^ (addr >> 16)) % map->bucket_count; + for (mi_pb_addr_entry_t* e = map->buckets[idx]; e != NULL; e = e->next) { + if (e->addr == addr) { *is_new = false; return e->id; } + } + mi_pb_addr_entry_t* e = mi_heap_malloc_tp(mi_pb_addr_entry_t, map->heap); + if (e == NULL) { map->failed = true; *is_new = false; return 0; } + e->addr = addr; + e->id = next_id; + e->next = map->buckets[idx]; + map->buckets[idx] = e; + *is_new = true; + return next_id; +} + +// Find the (1-based) `Mapping` id of the module containing `addr`, or 0 if +// `addr` doesn't fall within any collected module range. +static uint64_t mi_pprof_find_mapping_id(const mi_pprof_modules_t* mods, uintptr_t addr) { + for (size_t i = 0; i < mods->count; i++) { + if (addr >= mods->modules[i].start && addr < mods->modules[i].end) { return (uint64_t)(i + 1); } + } + return 0; +} + +// Build the current profiler data as a raw (uncompressed) `perftools.profiles.Profile` +// protobuf message in `profile` (see the file-level comment above). +// Returns `false` if any allocation failed; the caller releases all buffers in either case. +static bool mi_pprof_build_proto(mi_pprof_profiler_t* prof, mi_pb_buf_t* profile, mi_pb_strings_t* strings, + mi_pprof_modules_t* mods, mi_pb_addr_map_t* addr_map) +{ + mi_heap_t* heap = prof->profile_heap; + mi_locations_t* locations = &prof->locations; + + const int64_t str_count = mi_pb_strings_intern(strings, "count"); + const int64_t str_bytes = mi_pb_strings_intern(strings, "bytes"); + const int64_t str_alloc_objects = mi_pb_strings_intern(strings, "alloc_objects"); + const int64_t str_alloc_bytes = mi_pb_strings_intern(strings, "alloc_bytes"); + const int64_t str_inuse_objects = mi_pb_strings_intern(strings, "inuse_objects"); + const int64_t str_inuse_bytes = mi_pb_strings_intern(strings, "inuse_bytes"); + const int64_t str_space = mi_pb_strings_intern(strings, "space"); + const int64_t str_thread = mi_pb_strings_intern(strings, "thread"); + if (strings->failed) return false; + + // Profile.sample_type = 1: [alloc_objects:count, alloc_bytes:bytes, inuse_objects:count, inuse_bytes:bytes] + { + const int64_t types[4] = { str_alloc_objects, str_alloc_bytes, str_inuse_objects, str_inuse_bytes }; + const int64_t units[4] = { str_count, str_bytes, str_count, str_bytes }; + for (size_t i = 0; i < 4; i++) { + mi_pb_buf_t vt; + mi_pb_init(&vt, heap); + mi_pb_int64_field(&vt, 1, types[i]); + mi_pb_int64_field(&vt, 2, units[i]); + mi_pb_message_field(profile, 1, &vt); + mi_pb_done(&vt); + } + } + + // Profile.mapping = 3: the modules loaded into this process, so `pprof` can + // symbolize the (raw) addresses recorded in each `Location` below. + mi_pprof_collect_modules(heap, mods); + if (mods->failed) return false; + for (size_t i = 0; i < mods->count; i++) { + const mi_pprof_module_t* m = &mods->modules[i]; + mi_pb_buf_t mapping; + mi_pb_init(&mapping, heap); + mi_pb_uint64_field(&mapping, 1, (uint64_t)(i + 1)); // id + mi_pb_uint64_field(&mapping, 2, (uint64_t)m->start); // memory_start + mi_pb_uint64_field(&mapping, 3, (uint64_t)m->end); // memory_limit + if (m->path != NULL) { + mi_pb_int64_field(&mapping, 5, mi_pb_strings_intern(strings, m->path)); // filename + } + mi_pb_message_field(profile, 3, &mapping); + mi_pb_done(&mapping); + if (profile->failed || strings->failed) return false; + } + + // Profile.location = 4 and Profile.sample = 2: one (deduplicated) `Location` + // per unique frame address, and one `Sample` per unique call site (i.e. one + // per entry in our own locations hash table), exactly mirroring the text + // format's aggregation. + mi_pb_addr_map_init(addr_map, heap, locations->bucket_count); + if (addr_map->failed) return false; + uint64_t next_location_id = 1; + + for (size_t i = 0; i < locations->bucket_count; i++) { + for (mi_location_t* loc = mi_atomic_load_ptr_relaxed(mi_location_t, &locations->buckets[i]); loc != NULL; loc = mi_atomic_load_ptr_relaxed(mi_location_t, &loc->next)) { + const size_t inuse_count = mi_atomic_load_relaxed(&loc->inuse_count); + const size_t inuse_bytes = mi_atomic_load_relaxed(&loc->inuse_bytes); + + const size_t frame_count = (loc->callstack.count > MI_MAX_BACKTRACE_DEPTH ? MI_MAX_BACKTRACE_DEPTH : loc->callstack.count); + uint64_t location_ids[MI_MAX_BACKTRACE_DEPTH]; + for (size_t j = 0; j < frame_count; j++) { + const uintptr_t addr = (uintptr_t)loc->callstack.frames[j]; + bool is_new = false; + const uint64_t id = mi_pb_addr_map_find_or_insert(addr_map, addr, next_location_id, &is_new); + if (addr_map->failed) return false; + location_ids[j] = id; + if (is_new) { + next_location_id++; + mi_pb_buf_t location; + mi_pb_init(&location, heap); + mi_pb_uint64_field(&location, 1, id); // id + mi_pb_uint64_field(&location, 2, mi_pprof_find_mapping_id(mods, addr)); // mapping_id + mi_pb_uint64_field(&location, 3, (uint64_t)addr); // address + mi_pb_message_field(profile, 4, &location); + mi_pb_done(&location); + } + } + + mi_pb_buf_t sample; + mi_pb_init(&sample, heap); + mi_pb_packed_uint64_field(&sample, 1, location_ids, frame_count); // location_id (leaf-first, matching our capture order) + const int64_t values[4] = { + loc->alloc_count, loc->alloc_bytes, + (int64_t)inuse_count, (int64_t)inuse_bytes + }; + mi_pb_packed_int64_field(&sample, 2, values, 4); // value + { + // attach the originating thread id as a numeric label: pprof has no + // dedicated thread-id field, but `Sample.label` is the generic + // mechanism for this kind of per-sample metadata. + mi_pb_buf_t label; + mi_pb_init(&label, heap); + mi_pb_int64_field(&label, 1, str_thread); // key + mi_pb_int64_field(&label, 3, (int64_t)loc->thread_id); // num + mi_pb_message_field(&sample, 3, &label); + mi_pb_done(&label); + } + mi_pb_message_field(profile, 2, &sample); + mi_pb_done(&sample); + if (profile->failed) return false; + } + } + + // Profile.period_type = 11 / Profile.period = 12: matches the convention + // used by Go's heap profiles ("space"/"bytes", sampling period in bytes). + { + mi_pb_buf_t period_type; + mi_pb_init(&period_type, heap); + mi_pb_int64_field(&period_type, 1, str_space); + mi_pb_int64_field(&period_type, 2, str_bytes); + mi_pb_message_field(profile, 11, &period_type); + mi_pb_done(&period_type); + } + mi_pb_int64_field(profile, 12, (int64_t)prof->sample_threshold); // period + const int64_t elapsed_nanos = (int64_t)(_mi_clock_now() - prof->start_clock) * 1000000LL; // clock is in milli-secs + mi_pb_int64_field(profile, 9, prof->start_time_secs * 1000000000LL + elapsed_nanos); // time_nanos + mi_pb_int64_field(profile, 10, elapsed_nanos); // duration_nanos + + // the string table (Profile.string_table = 6) can only be finalized now, + // since strings were interned into it while building the message above. + if (strings->failed) return false; + mi_pb_bytes(profile, strings->table.data, strings->table.size); + if (profile->failed) return false; + return true; +} + +static bool mi_pprof_write_file(const char* fname, const mi_pb_buf_t* profile) { + FILE* f = mi_pprof_fopen(fname, true); + if (f == NULL) return false; + bool ok = (fwrite(profile->data, 1, profile->size, f) == profile->size); + if (fclose(f) != 0) { ok = false; } + return ok; +} + +// Write out the current profiler data as a protobuf `Profile` message. +// Returns `false` if any allocation failed (in which case no file is written): +// the snapshot is best effort and must never crash the profiled program. +static bool mi_pprof_snapshot_proto(mi_pprof_profiler_t* prof, const char* fname) { + mi_heap_t* heap = prof->profile_heap; + mi_pb_buf_t profile; + mi_pb_init(&profile, heap); + mi_pb_strings_t strings; + mi_pb_strings_init(&strings, heap); + mi_pprof_modules_t mods; + mods.modules = NULL; + mods.count = 0; + mods.capacity = 0; + mods.failed = false; + mi_pb_addr_map_t addr_map; + addr_map.heap = heap; + addr_map.bucket_count = 0; + addr_map.buckets = NULL; + addr_map.failed = false; + + bool ok = mi_pprof_build_proto(prof, &profile, &strings, &mods, &addr_map); + mi_pb_addr_map_done(&addr_map); + mi_pprof_modules_done(&mods); + mi_pb_strings_done(&strings); + if (ok) { ok = mi_pprof_write_file(fname, &profile); } + mi_pb_done(&profile); + return ok; +} + +#endif // MI_PROFILE diff --git a/system/lib/mimalloc/src/profile/readme.md b/system/lib/mimalloc/src/profile/readme.md new file mode 100644 index 0000000000000..581a5f58ab1a8 --- /dev/null +++ b/system/lib/mimalloc/src/profile/readme.md @@ -0,0 +1,5 @@ +# Allocation Sampling and Profiling + +- [`sample-profile.c`](sample-profile.c): Allocation sampling, profiler callbacks. +- [`sample-guarded.c`](sample-guarded.c): Sampled allocations protected by guard pages. +- [`pprof.c`](pprof.c): Default heap profiler implementation with stack traces and pprof-compatible text or protobuf snapshots. diff --git a/system/lib/mimalloc/src/sample-guarded.c b/system/lib/mimalloc/src/profile/sample-guarded.c similarity index 82% rename from system/lib/mimalloc/src/sample-guarded.c rename to system/lib/mimalloc/src/profile/sample-guarded.c index f7c1fe81e99ba..a1d8b34823658 100644 --- a/system/lib/mimalloc/src/sample-guarded.c +++ b/system/lib/mimalloc/src/profile/sample-guarded.c @@ -5,8 +5,10 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc/internal.h" -#include "mimalloc/prim-tls.h" +#include "mimalloc/prim/prim-tls.h" +#include "mimalloc/util/stats.h" +#include "mimalloc/alloc/page.h" +#include "mimalloc/profile/sample-guarded.h" mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) { theap->guarded_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : sample_rate); @@ -17,8 +19,8 @@ mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t s theap->guarded_sample_countdown = (seed % theap->guarded_sample_rate) + 1; // start at random count between 1 and `sample_rate` } else if (theap->guarded_sample_countdown > theap->guarded_sample_rate) { - theap->guarded_sample_countdown = theap->guarded_sample_rate; - } + theap->guarded_sample_countdown = theap->guarded_sample_rate; + } _mi_theap_update_sample_rate(theap); } @@ -42,11 +44,11 @@ void _mi_theap_guarded_init(mi_theap_t* theap) { // We always allocate a guarded allocation at an offset (`mi_page_has_interior_pointers` will be true). -// We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`) +// We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc/alloc-aligned.c`) // and the first word to `~0` for guarded allocations to have a correct `mi_usable_size` static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size, size_t* usable_size) { // todo: we can still make padding work by moving it out of the guard page area - mi_page_t* const page = _mi_ptr_page(block); + mi_page_t* const page = mi_ptr_page(block); mi_page_set_has_interior_pointers(page, true); block->next = MI_BLOCK_TAG_GUARDED; @@ -61,10 +63,10 @@ static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size, size_t } uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size; // note: the alignment of the guard page relies on blocks being os_page_size aligned which - // is ensured in `mi_arena_page_alloc_fresh`. - mi_assert_internal(_mi_is_aligned(block, os_page_size)); - mi_assert_internal(_mi_is_aligned(guard_page, os_page_size)); - if (!page->memid.is_pinned && _mi_is_aligned(guard_page, os_page_size)) { + // is ensured in `mi_arena_page_alloc_fresh`. + mi_assert_internal(mi_is_aligned(block, os_page_size)); + mi_assert_internal(mi_is_aligned(guard_page, os_page_size)); + if (!page->memid.is_pinned && mi_is_aligned(guard_page, os_page_size)) { const bool ok = _mi_os_protect(guard_page, os_page_size); if mi_unlikely(!ok) { _mi_warning_message("failed to set a guard page behind an object (object %p of size %zu)\n", block, block_size); @@ -94,14 +96,14 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, { // allocate multiple of page size ending in a guard page // ensure minimal alignment requirement? - if mi_unlikely(size >= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { // check up front so the `req_size` won't overflow + if mi_unlikely(size >= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { // check up front so the `req_size` won't overflow _mi_error_message(EOVERFLOW, "(guarded) allocation request is too large (%zu bytes)\n", size); return NULL; } const size_t os_page_size = _mi_os_page_size(); - const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE)); - const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE); - const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size); + const size_t obj_size = (mi_option_is_enabled(mi_option_unsafe_guarded_precise) ? size : mi_align_up(size, MI_MAX_ALIGN_SIZE)); + const size_t bsize = mi_align_up(mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE); + const size_t req_size = mi_align_up(bsize + os_page_size, os_page_size); // const size_t threshold = mi_theap_disable_profiler(theap); mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap, req_size, false /* don't zero */, ppage); // mi_theap_enable_profiler(theap,threshold); @@ -110,11 +112,11 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, void* const p = mi_block_ptr_set_guarded(block, obj_size, &usable_size); if (p == NULL) return NULL; if (zero) { - _mi_memzero(p,obj_size); // we have to zero afterwards as padding might have written inside the block (if the `blocksize > reqsize + os_page_size`) + mi_memzero(p,obj_size); // we have to zero afterwards as padding might have written inside the block (if the `blocksize > reqsize + os_page_size`) } // stats - mi_track_malloc(p, usable_size, zero); + mi_track_malloc(p, usable_size, zero); if (!mi_theap_is_initialized(theap)) { theap = _mi_theap_default(); } mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1); #if MI_STATS @@ -129,9 +131,9 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, #endif #if MI_PAGE_META_SMALL_IS_ALIGNED && MI_DEBUG>=2 // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail. - if (size <= MI_SMALL_SIZE_MAX) { - mi_page_t* const page = _mi_ptr_page(p); - mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); + if (size <= MI_SMALL_SIZE_MAX) { + mi_page_t* const page = mi_ptr_page(p); + mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); } #endif return p; @@ -142,6 +144,7 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, #if MI_GUARDED void _mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { // if (!mi_block_ptr_is_guarded(block,p)) return; + MI_UNUSED_RELEASE(p); mi_assert_internal(mi_block_ptr_is_guarded(block, p)); mi_assert_internal(mi_page_has_interior_pointers(page)); mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t)); @@ -150,15 +153,15 @@ void _mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { const size_t bsize = mi_page_block_size(page); const size_t psize = _mi_os_page_size(); mi_assert_internal(bsize > psize); - mi_assert_internal(!page->memid.is_pinned); + if (page->memid.is_pinned) return; // no guard page was set (see `mi_block_ptr_set_guarded`) void* gpage = (uint8_t*)block + bsize - psize; - mi_assert_internal(_mi_is_aligned(gpage, psize)); + mi_assert_internal(mi_is_aligned(gpage, psize)); _mi_os_unprotect(gpage, psize); } // unguard a whole page (called from `mi_heap_destroy`) -void _mi_page_unguard_all(mi_page_t* page) { - if mi_likely(!mi_page_has_interior_pointers(page)) return; +void _mi_page_unguard_all(mi_page_t* page) { + if mi_likely(!mi_page_has_interior_pointers(page) || page->memid.is_pinned) return; uint8_t* const start = mi_page_start(page); const size_t psize = mi_page_committed(page); _mi_os_unprotect(start,psize); // unprotect all at once as we cannot know which blocks are guarded diff --git a/system/lib/mimalloc/src/sample-profile.c b/system/lib/mimalloc/src/profile/sample-profile.c similarity index 80% rename from system/lib/mimalloc/src/sample-profile.c rename to system/lib/mimalloc/src/profile/sample-profile.c index f1b5db4913e3e..ec1bf1c3fcaba 100644 --- a/system/lib/mimalloc/src/sample-profile.c +++ b/system/lib/mimalloc/src/profile/sample-profile.c @@ -5,11 +5,14 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc/internal.h" -#include "mimalloc/prim-tls.h" +#include "mimalloc/prim/prim-tls.h" +#include "mimalloc/util/stats.h" +#include "mimalloc/alloc/page.h" +#include "mimalloc/profile/sample-guarded.h" +#include "mimalloc/profile/sample-profile.h" //---------------------------------------------------------------------------- -// General sampled allocation (called from `page.c:mi_malloc_generic_fallback`) +// General sampled allocation (called from `alloc/alloc-generic.c:mi_malloc_generic_fallback`) // This is for both guarded and profiled sampling //---------------------------------------------------------------------------- @@ -26,28 +29,32 @@ size_t _mi_theap_update_sample_rate(mi_theap_t* theap) { } // Called when the theap sample countdown <= req_size; should call guarded or profiled allocation. -mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage) mi_attr_noexcept +mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage) mi_attr_noexcept { // the size has not yet been counted against the countdown (and does not include MI_PADDING_SIZE) mi_assert_internal(req_size <= MI_MAX_ALLOC_SIZE); - mi_assert_internal((theap->sample_countdown==SIZE_MAX && _mi_is_empty_theap(theap)) || + mi_assert_internal((theap->sample_countdown==SIZE_MAX && _mi_is_empty_theap(theap)) || (theap->sample_countdown <= MI_SAMPLE_COUNTDOWN_MAX && theap->sample_countdown < req_size)); const size_t size = req_size + MI_PADDING_SIZE; // handle empty theap and disabled sampling - if (theap->sample_rate==0) { - if (!_mi_is_empty_theap(theap)) { // avoid writing to the initial empty theap - theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // avoid the sampling path for a long time + if (theap->sample_rate==0) { + if (!_mi_is_empty_theap(theap)) { // avoid writing to the initial empty theap + theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // avoid the sampling path for a long time } return _mi_malloc_generic_no_sample(theap,size,zero,ppage); } - + // never sample allocations made by a profiler `on_alloc` callback (e.g. by `backtrace`) to avoid recursion + if mi_unlikely(theap->tld->in_profiler) { + return _mi_malloc_generic_no_sample(theap,size,zero,ppage); + } + // update countdown and total accummulated requested bytes since the last sample - mi_assert_internal(!_mi_is_empty_theap(theap)); + mi_assert_internal(!_mi_is_empty_theap(theap)); mi_assert_internal(req_size <= SIZE_MAX/2); mi_assert_internal(theap->sample_rate > 0); mi_assert_internal(theap->sample_rate <= SIZE_MAX/2); - mi_assert_internal(theap->sample_rate >= theap->sample_countdown); + mi_assert_internal(theap->sample_rate >= theap->sample_countdown); const uint64_t requested = theap->sample_requested = (uint64_t)theap->sample_rate + (uint64_t)(req_size - theap->sample_countdown) + theap->sample_requested; mi_assert_internal(requested > 0); @@ -102,10 +109,10 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* the static mi_profiler_t* mi_heap_profiler(const mi_heap_t* heap) { return mi_atomic_load_ptr_acquire(mi_profiler_t,&heap->profiler); -} +} static mi_profiler_t* mi_theap_get_enabled_profiler(const mi_theap_t* theap) { - mi_heap_t* const heap = _mi_theap_heap(theap); + mi_heap_t* const heap = mi_theap_heap(theap); mi_profiler_t* prof = mi_atomic_load_ptr_relaxed(mi_profiler_t, &heap->profiler); if (prof!=NULL && mi_profiler_is_enabled(prof) && prof->on_alloc != NULL) { return prof; @@ -116,43 +123,52 @@ static mi_profiler_t* mi_theap_get_enabled_profiler(const mi_theap_t* theap) { } //---------------------------------------------------------------------------- -// Profile an allocation +// Profile an allocation //----------------------------------------------------------------------------- +// call `on_alloc`; allocations made by the profiler itself (e.g. by `backtrace`) are not sampled +static size_t mi_profiler_on_alloc(mi_theap_t* theap, mi_profiler_t* prof, mi_profiler_sample_data_t* data, void* p, size_t req_size, uint64_t requested_since_last_sample) { + mi_tld_t* const tld = theap->tld; + tld->in_profiler = true; + const size_t new_sample_rate = (*prof->on_alloc)(prof, data, p, req_size, theap->profile_sample_rate, requested_since_last_sample, mi_theap_heap(theap)); + tld->in_profiler = false; + return new_sample_rate; +} + mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, uint64_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept { - mi_assert_internal(theap!=NULL); + mi_assert_internal(theap!=NULL); mi_assert_internal(size>=MI_PADDING_SIZE); const size_t req_size = size - MI_PADDING_SIZE; mi_assert_internal(req_size<=requested_since_last_sample); mi_profiler_t* const prof = mi_theap_get_enabled_profiler(theap); if (prof == NULL) { return _mi_malloc_generic_no_sample(theap,size,zero,ppage); } mi_assert_internal(prof!=NULL && prof->on_alloc!=NULL && mi_profiler_is_enabled(prof)); - + void* p = NULL; size_t new_sample_rate = 0; - if (prof->on_free==NULL || prof->sample_data_size==0) { + if (prof->on_free==NULL || prof->sample_data_size==0) { // just allocate without profiler data p = _mi_malloc_generic_no_sample(theap,size,zero,ppage); if (p==NULL) { return p; } if (prof->on_alloc!=NULL) { - // we are just allocation profiling (not heap profiling as on_free == NULL) - new_sample_rate = (*prof->on_alloc)(prof, NULL /* no data */, p, req_size, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap) ); + // we are just allocation profiling (not heap profiling as on_free == NULL) + new_sample_rate = mi_profiler_on_alloc(theap, prof, NULL /* no data */, p, req_size, requested_since_last_sample); } } else { // overallocate a larger block to store the profiler data // [MI_BLOCK_TAG_PROFILE] [usable size] [ ... profile data ... ] [... user data ...] const size_t sample_data_offset = sizeof(mi_block_t); - const size_t sample_user_data_size = _mi_align_up(prof->sample_data_size > MI_PROFILE_SAMPLE_DATA_MAX_SIZE ? MI_PROFILE_SAMPLE_DATA_MAX_SIZE : prof->sample_data_size, sizeof(void*)); + const size_t sample_user_data_size = mi_align_up(prof->sample_data_size > MI_PROFILE_SAMPLE_DATA_MAX_SIZE ? MI_PROFILE_SAMPLE_DATA_MAX_SIZE : prof->sample_data_size, sizeof(void*)); const size_t sample_data_size = sizeof(mi_profiler_sample_data_t) + sample_user_data_size; // one void* too many just in case - const size_t user_offset = _mi_align_up(sample_data_offset + sample_data_size, MI_MAX_ALIGN_SIZE); + const size_t user_offset = mi_align_up(sample_data_offset + sample_data_size, MI_MAX_ALIGN_SIZE); const size_t oversize = user_offset + size; mi_page_t* page = NULL; - mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap,oversize,zero,&page); + mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap,oversize,zero,&page); if (block==NULL) return NULL; mi_assert_internal(page!=NULL); - if (ppage!=NULL) { *ppage = page; } + if (ppage!=NULL) { *ppage = page; } #if MI_PAGE_META_SMALL_IS_ALIGNED // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail. // (that is why we need to limit the profiler_data_size as well) @@ -162,20 +178,20 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th // Set up the profiled block as an interior pointer so the interior "slow" path is taken on mi_free (where we catch it to call on_free) mi_page_set_has_interior_pointers(page, true); - block->next = MI_BLOCK_TAG_PROFILED; + block->next = MI_BLOCK_TAG_PROFILED; p = (uint8_t*)block + user_offset; mi_profiler_sample_data_t* const sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + sample_data_offset); sample_data->user_data_size = sample_user_data_size; // and call the profiler on_alloc - if (prof->on_alloc!=NULL) { - new_sample_rate = (*prof->on_alloc)(prof, sample_data, p, req_size, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap) ); + if (prof->on_alloc!=NULL) { + new_sample_rate = mi_profiler_on_alloc(theap, prof, sample_data, p, req_size, requested_since_last_sample); } } - if (new_sample_rate!=0 && new_sample_rate != (size_t)theap->profile_sample_rate) { + if (new_sample_rate!=0 && new_sample_rate != (size_t)theap->profile_sample_rate) { _mi_theap_set_profile_sample_rate(theap,new_sample_rate); } - mi_theap_stat_counter_increase(theap,profile_samples,1); + mi_theap_stat_counter_increase(theap,profile_samples,1); return p; } @@ -187,7 +203,7 @@ void _mi_page_profile_on_free(mi_page_t* page, mi_block_t* block, void* p) { if (heap==NULL) return; mi_profiler_t* prof = mi_heap_profiler(heap); if (prof==NULL || !mi_profiler_is_enabled(prof) || prof->on_free==NULL) return; - + // call the on_free callback mi_profiler_sample_data_t* const sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + sizeof(mi_block_t)); (*prof->on_free)(prof, sample_data, p, heap); @@ -201,7 +217,7 @@ void _mi_page_profile_on_free(mi_page_t* page, mi_block_t* block, void* p) { size_t _mi_theap_set_profile_sample_rate(mi_theap_t* theap, size_t sample_rate) { const size_t old_sample_rate = theap->profile_sample_rate; theap->profile_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : sample_rate); - if (theap->profile_sample_countdown > theap->profile_sample_rate) { + if (theap->profile_sample_countdown > theap->profile_sample_rate) { theap->profile_sample_countdown = theap->profile_sample_rate; // todo: adjust difference? } _mi_theap_update_sample_rate(theap); @@ -219,17 +235,17 @@ mi_decl_export bool mi_heap_profile(mi_heap_t* heap, mi_profiler_t* profiler) { } mi_decl_export void mi_heap_profile_disable(mi_heap_t* heap) { - static mi_profiler_t mi_nosample_profiler = { NULL, 0, MI_SAMPLE_RATE_MAX, NULL, NULL, NULL }; + static mi_profiler_t mi_nosample_profiler = { NULL, 0, MI_SAMPLE_RATE_MAX, NULL, NULL, NULL, NULL }; mi_heap_set_profiler(heap,NULL); // first clear any previous one mi_heap_set_profiler(heap,&mi_nosample_profiler); // and then install the nosample one } mi_decl_export bool mi_subproc_profile(mi_subproc_id_t subproc_id, mi_profiler_t* profiler) { mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); - if (subproc==NULL) return false; - mi_profiler_stop(profiler); + if (subproc==NULL) return false; + mi_profiler_stop(profiler); mi_profiler_t* previous = (profiler==NULL ? mi_atomic_load_ptr_acquire(mi_profiler_t,&subproc->profiler) : NULL); // don't overwrite unless it is NULL - if (!mi_atomic_cas_ptr_strong_acq_rel(mi_profiler_t,&subproc->profiler, &previous, profiler)) { return false; } + if (!mi_atomic_cas_ptr_strong_acq_rel(mi_profiler_t,&subproc->profiler, &previous, profiler)) { return false; } mi_lock(&subproc->heaps_lock) { for (mi_heap_t* heap = subproc->heaps; heap!=NULL; heap = heap->next) { mi_heap_set_profiler(heap,profiler); @@ -244,7 +260,7 @@ mi_decl_export bool mi_profile( mi_profiler_t* profiler) { bool mi_profiler_start(mi_profiler_t* profiler ) { if (profiler==NULL) return false; - const bool was_running = mi_profiler_set_enabled(profiler,true); + const bool was_running = mi_profiler_set_enabled(profiler,true); if (was_running) return true; // for the main heap, if this is the profiler, start the theap more aggressively // otherwise it will be picked up when theaps take the slow generic malloc path. @@ -259,6 +275,15 @@ bool mi_profiler_start(mi_profiler_t* profiler ) { } bool mi_profiler_stop(mi_profiler_t* profiler) { - if (profiler==NULL) return true; + if (profiler==NULL) return false; return mi_profiler_set_enabled(profiler,false); } + +void mi_profiler_snapshot(mi_profiler_t* profiler) { + if (profiler==NULL) return; + if (profiler->on_snapshot==NULL) return; + static mi_atomic_guard_t guard; + mi_atomic_guard(&guard) { + (*profiler->on_snapshot)(profiler); + } +} diff --git a/system/lib/mimalloc/src/readme.md b/system/lib/mimalloc/src/readme.md new file mode 100644 index 0000000000000..4902af4189271 --- /dev/null +++ b/system/lib/mimalloc/src/readme.md @@ -0,0 +1,16 @@ +# Sources + +[`prim`](/include/mimalloc/prim.h) → [`os`](/include/mimalloc/prim/os.h) → [`arena`](/include/mimalloc/arena/arena-page.h`) → [`page`](/include/mimalloc/alloc/page.h) → [`alloc`](alloc/alloc.c) + +* [`alloc`](alloc): "front-end": main allocation and free functions. + +* [`arena`](arena): "back-end": management of large memory arenas allocated from the OS. + +* [`heap`](heap): initialization, first-class heaps, thread local _theaps_, and subprocesses. + +* [`prim`](prim): primitive functions needed from the host, and OS level allocations. + +* [`profile`](profile): sampling for guarded allocations, profiled allocations, and the default profiler. + +* [`util`](util): utility functions, libc, options, random numbers, statistics, and dynamic thread locals. + diff --git a/system/lib/mimalloc/src/static.c b/system/lib/mimalloc/src/static.c index 8a4f8c3d54052..edf493c6b423d 100644 --- a/system/lib/mimalloc/src/static.c +++ b/system/lib/mimalloc/src/static.c @@ -1,46 +1,65 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2020, Microsoft Research, Daan Leijen +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ + +// -------------------------------------------------------------------------- +// This is a "single file include" of the entire mimalloc library. +// Just include this file in your build to link statically with mimalloc. +// This will also override the standard library allocation functions +// (if cmake -DMI_OVERRIDE=ON (or the macro MI_MALLOC_OVERRIDE=1 is defined)). +// +// This is also used for a static override where we create a single object +// file containing the whole library. +// If such object file is linked first it will override all the standard library +// allocation functions (on Unix's). +// -------------------------------------------------------------------------- + #ifndef _DEFAULT_SOURCE #define _DEFAULT_SOURCE #endif #if defined(__sun) -// same remarks as os.c for the static's context. +// same remarks as prim/os.c for the static's context. #undef _XOPEN_SOURCE #undef _POSIX_C_SOURCE #endif #include "mimalloc.h" -#include "mimalloc/internal.h" - -// For a static override we create a single object file -// containing the whole library. If it is linked first -// it will override all the standard library allocation -// functions (on Unix's). -#include "alloc.c" // includes alloc-override.c and free.c -#include "alloc-aligned.c" -#include "alloc-posix.c" -#include "arena.c" -#include "bitmap.c" -#include "heap.c" -#include "init.c" -#include "libc.c" -#include "options.c" -#include "os.c" -#include "prim/prim.c" // includes platform specific prim//prim.c + +#include "prim/os.c" +#include "prim/prim.c" // includes platform specific prim//prim.c #include "prim/prim-tls.c" #if MI_OSX_ZONE #include "prim/osx/alloc-override-zone.c" #endif -#include "page.c" // includes page-queue.c -#include "page-map.c" -#include "random.c" -#include "sample-guarded.c" -#include "sample-profile.c" -#include "stats.c" -#include "subproc.c" -#include "theap.c" -#include "threadlocal.c" + +#include "util/libc.c" +#include "util/options.c" +#include "util/random.c" +#include "util/stats.c" +#include "util/threadlocal.c" + +#include "arena/arena.c" +#include "arena/arena-alloc.c" +#include "arena/arena-page.c" +#include "arena/bitmap-chunk.c" +#include "arena/bitmap.c" + +#include "alloc/alloc.c" // includes alloc-override.c and free.c (for aliasing to work) +#include "alloc/alloc-generic.c" +#include "alloc/alloc-aligned.c" +#include "alloc/alloc-posix.c" +#include "alloc/page.c" +#include "alloc/page-queue.c" + +#include "heap/heap.c" +#include "heap/init.c" +#include "heap/page-map.c" +#include "heap/subproc.c" +#include "heap/theap.c" + +#include "profile/sample-guarded.c" +#include "profile/sample-profile.c" +#include "profile/pprof.c" diff --git a/system/lib/mimalloc/src/libc.c b/system/lib/mimalloc/src/util/libc.c similarity index 82% rename from system/lib/mimalloc/src/libc.c rename to system/lib/mimalloc/src/util/libc.c index 8f211230d0cd3..ef4d22966fd00 100644 --- a/system/lib/mimalloc/src/libc.c +++ b/system/lib/mimalloc/src/util/libc.c @@ -13,8 +13,19 @@ terms of the MIT license. A copy of the license can be found in the file // -------------------------------------------------------- #include "mimalloc.h" -#include "mimalloc/internal.h" -#include "mimalloc/prim.h" // mi_prim_getenv +#include "mimalloc/prim.h" // mi_prim_getenv +#include "mimalloc/prim/prim-tls.h" +#include "mimalloc/util/options.h" +#include "mimalloc/util/libc.h" +#include // LONG_MIN, LONG_MAX + +#if MI_DEBUG && NDEBUG +#if defined(_MSC_VER) +#pragma message("mimalloc assertions enabled in a release build") +#else +#warning "mimalloc assertions enabled in a release build" +#endif +#endif char _mi_toupper(char c) { if (c >= 'a' && c <= 'z') return (c - 'a' + 'A'); @@ -88,6 +99,81 @@ char* _mi_strnstr(char* s, size_t max_len, const char* pat) { return NULL; } +const char* _mi_strchr(const char* s, char c) { + if (s==NULL) return NULL; + for (; *s != 0; s++) { + if (*s == c) return s; + } + return (c == 0 ? s : NULL); +} + +// backward search: like `strrchr`, returns the last occurrence of `c` in `s`, or NULL if not found. +const char* _mi_strrchr(const char* s, char c) { + if (s==NULL) return NULL; + const char* last = (c == 0 ? s + _mi_strlen(s) : NULL); + for (; *s != 0; s++) { + if (*s == c) last = s; + } + return last; +} + + +static bool mi_isspace(char c) { + return (c == ' ' || (c >= '\t' && c <= '\r')); +} + +static int mi_digit_value(char c, int base) { + if (c >= '0' && c <= '9') return (c - '0'); + if (base == 16) { + if (c >= 'a' && c <= 'f') return (c - 'a' + 10); + if (c >= 'A' && c <= 'F') return (c - 'A' + 10); + } + return -1; +} + +// Parse up to max_len bytes without errno or requiring a terminator; skip at most 128 whitespace characters. +// Allow at most 64 digits; leading zeros are allowed only in hexadecimal. +// On failure, result is zero (clamped on overflow); end is s if no digits, otherwise the stopping point. +bool _mi_strntol(const char* s, size_t max_len, const char** end, long* result) { + if (end != NULL) { *end = s; } + if (result != NULL) { *result = 0; } + if (s == NULL) return false; + size_t i = 0; + while (i < max_len && i < 128 && mi_isspace(s[i])) { i++; } + if (i == max_len) return false; + const bool neg = (s[i] == '-'); + if (neg || s[i] == '+') { i++; } + if (i == max_len) return false; + int base = 10; + if (i + 1 < max_len && s[i] == '0' && (s[i+1] == 'x' || s[i+1] == 'X')) { + base = 16; + i += 2; + } + if (base == 10 && i + 1 < max_len && s[i] == '0' && mi_digit_value(s[i+1], base) >= 0) return false; + const size_t start = i; + // Accumulate negatively so LONG_MIN is representable without unsigned arithmetic. + const long limit = (neg ? LONG_MIN : -LONG_MAX); + const long cutoff = limit / base; + const long cutlim = -(limit % base); + long acc = 0; + while (i < max_len && i - start < 64) { + const int digit = mi_digit_value(s[i], base); + if (digit < 0) break; + if (acc < cutoff || (acc == cutoff && digit > cutlim)) { + if (end != NULL) { *end = s + i; } + if (result != NULL) { *result = (neg ? LONG_MIN : LONG_MAX); } + return false; + } + acc = acc*base - digit; + i++; + } + if (i == start) return false; + if (end != NULL) { *end = s + i; } + if (i < max_len && i - start == 64 && mi_digit_value(s[i], base) >= 0) return false; + if (result != NULL) { *result = (neg ? acc : -acc); } + return true; +} + #ifdef MI_NO_GETENV int _mi_getenv(const char* name, char* result, size_t result_size) { MI_UNUSED(name); @@ -211,7 +297,7 @@ void _mi_detect_cpu_features(void) { // initialized (and to reduce dependencies) // // format: d i, p x u, s -// prec: z l ll L +// prec: z l ll L l8 l4 // l8=64-bit, l4=32-bit // width: 10 // align-left: - // fill: 0 @@ -264,7 +350,7 @@ static void mi_out_num(uintmax_t x, size_t base, char prefix, char** out, char* } else { #define MI_MAX_OUT_DIGITS (160) /* a 512 bit number has 155 digits */ - char num[MI_MAX_OUT_DIGITS]; + char num[MI_MAX_OUT_DIGITS]; int dcount = 0; while(x>0 && dcount < MI_MAX_OUT_DIGITS) { char digit = (char)(x % base); @@ -330,12 +416,17 @@ int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) { } if (c == 0) break; // extra check due to while } - if (c == 'z' || c == 't' || c == 'L') { numtype = c; MI_NEXTC(); } + if (c == 'z' || c == 't' || c == 'L') { + numtype = c; MI_NEXTC(); + } else if (c == 'l') { numtype = c; MI_NEXTC(); if (c == 'l') { numtype = 'L'; MI_NEXTC(); } + else if (c == '8') { numtype = '8'; MI_NEXTC(); } // 64-bit + else if (c == '4') { numtype = '4'; MI_NEXTC(); } // 32-bit } + char* start = out; if (c == '%') { mi_outc('%', &out, end); @@ -353,6 +444,8 @@ int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) { else if (numtype == 't') x = va_arg(args, uintptr_t); // unsigned ptrdiff_t else if (numtype == 'L') x = va_arg(args, unsigned long long); else if (numtype == 'l') x = va_arg(args, unsigned long); + else if (numtype == '8') x = va_arg(args, uint64_t); + else if (numtype == '4') x = va_arg(args, uint32_t); else x = va_arg(args, unsigned int); } else if (c == 'p') { @@ -376,6 +469,8 @@ int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) { else if (numtype == 't') x = va_arg(args, ptrdiff_t); else if (numtype == 'L') x = va_arg(args, long long); else if (numtype == 'l') x = va_arg(args, long); + else if (numtype == '8') x = va_arg(args, int64_t); + else if (numtype == '4') x = va_arg(args, int32_t); else x = va_arg(args, int); char pre = 0; if (x < 0) { @@ -543,4 +638,3 @@ mi_decl_noinline size_t _mi_popcount_generic(size_t x) { return mi_popcount_generic64(x); } #endif - diff --git a/system/lib/mimalloc/src/options.c b/system/lib/mimalloc/src/util/options.c similarity index 86% rename from system/lib/mimalloc/src/options.c rename to system/lib/mimalloc/src/util/options.c index eaa2119def5fc..6489ffbbcf5a6 100644 --- a/system/lib/mimalloc/src/options.c +++ b/system/lib/mimalloc/src/util/options.c @@ -5,9 +5,10 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc/internal.h" -#include "mimalloc/atomic.h" #include "mimalloc/prim.h" // mi_prim_out_stderr +#include "mimalloc/prim/prim-tls.h" +#include "mimalloc/alloc/page.h" +#include "mimalloc/util/options.h" #include // stdin/stdout #include // abort @@ -96,6 +97,10 @@ int mi_version(void) { #define MI_DEFAULT_PAGE_MAX_RECLAIM (-1) // unlimited #endif +#ifndef MI_DEFAULT_PAGE_FULL_RETAIN +#define MI_DEFAULT_PAGE_FULL_RETAIN 2 +#endif + #ifndef MI_DEFAULT_PAGE_CROSS_THREAD_MAX_RECLAIM #define MI_DEFAULT_PAGE_CROSS_THREAD_MAX_RECLAIM 32 #endif @@ -109,8 +114,16 @@ int mi_version(void) { #endif #endif -#ifndef MI_DEFAULT_COLLECT_MERGES_STATS -#define MI_DEFAULT_COLLECT_MERGES_STATS 1 +#ifndef MI_DEFAULT_STATS_MERGE_ON_COLLECT +#define MI_DEFAULT_STATS_MERGE_ON_COLLECT 1 +#endif + +#ifndef MI_DEFAULT_STATS_MERGE_THRESHOLD +#define MI_DEFAULT_STATS_MERGE_THRESHOLD (512*1024) // 512 KiB +#endif + +#ifndef MI_DEFAULT_ARENA_PURGE_IMMEDIATE_SIZE +#define MI_DEFAULT_ARENA_PURGE_IMMEDIATE_SIZE ((MI_SIZE_SIZE * MI_LARGE_PAGE_SIZE) / MI_KiB) // 32 MiB (in KiB) #endif // Static options @@ -158,13 +171,14 @@ static mi_option_desc_t mi_options[_mi_option_last] = { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_visit_abandoned) }, { 0, MI_OPTION_UNINIT, MI_OPTION(guarded_min) }, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects { MI_GiB, MI_OPTION_UNINIT, MI_OPTION(guarded_max) }, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects - { 0, MI_OPTION_UNINIT, MI_OPTION(guarded_precise) }, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0) + { 0, MI_OPTION_UNINIT, MI_OPTION(unsafe_guarded_precise) }, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0) { MI_DEFAULT_GUARDED_SAMPLE_RATE, - MI_OPTION_UNINIT, MI_OPTION(guarded_sample_rate)}, // 1 out of N allocations in the min/max range will be guarded (=4000) + MI_OPTION_UNINIT, MI_OPTION(guarded_sample_rate)}, // guard an object in the min/max range about every N requested bytes (=0) { 0, MI_OPTION_UNINIT, MI_OPTION(guarded_sample_seed)}, { 10000, MI_OPTION_UNINIT, MI_OPTION(generic_collect) }, // collect theaps every N (=10000) generic allocation calls - { 0, MI_OPTION_UNINIT, MI_OPTION_LEGACY(page_reclaim_on_free, abandoned_reclaim_on_free) },// reclaim abandoned (small) pages on a free: -1 = disable completely, 0 = only reclaim into the originating theap, 1 = reclaim on free across theaps - { 2, MI_OPTION_UNINIT, MI_OPTION(page_full_retain) }, // number of (small) pages to retain in the free page queues + { 1, MI_OPTION_UNINIT, MI_OPTION_LEGACY(page_reclaim_on_free, abandoned_reclaim_on_free) },// reclaim abandoned (small) pages on a free: -1 = disable completely, 0 = only reclaim into the originating theap, 1 = reclaim on free across theaps + { MI_DEFAULT_PAGE_FULL_RETAIN, + MI_OPTION_UNINIT, MI_OPTION(page_full_retain) }, // number of (small) pages to retain in the free page queues { 4, MI_OPTION_UNINIT, MI_OPTION(page_max_candidates) }, // max search to find a best page candidate { 0, MI_OPTION_UNINIT, MI_OPTION(max_vabits) }, // max virtual address space bits { MI_DEFAULT_PAGEMAP_COMMIT, @@ -176,19 +190,39 @@ static mi_option_desc_t mi_options[_mi_option_last] = MI_OPTION_UNINIT, MI_OPTION(page_cross_thread_max_reclaim) }, // don't reclaim (small) pages across threads if we already own N pages in that size class { MI_DEFAULT_ALLOW_THP, MI_OPTION_UNINIT, MI_OPTION(allow_thp) }, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process. - { 0, MI_OPTION_UNINIT, MI_OPTION(minimal_purge_size) }, // set minimal purge size (in KiB) (=0). Using 0 resolves to either 64 (or 2048 if THP is enabled). + { 0, MI_OPTION_UNINIT, MI_OPTION_LEGACY(arena_purge_min_size, minimal_purge_size) }, // set minimal purge size (in KiB) (=0). Using 0 resolves to either 64 (or 2048 if THP is enabled). { MI_DEFAULT_ARENA_MAX_OBJECT_SIZE, MI_OPTION_UNINIT, MI_OPTION(arena_max_object_size) }, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit). { 0, MI_OPTION_UNINIT, MI_OPTION(arena_is_numa_local) }, // associate local numa node with an initial arena allocation - { MI_DEFAULT_COLLECT_MERGES_STATS, - MI_OPTION_UNINIT, MI_OPTION(collect_merges_stats) }, // on each theap collect, stats are merged with the parent heap + { MI_DEFAULT_STATS_MERGE_ON_COLLECT, + MI_OPTION_UNINIT, MI_OPTION(stats_merge_on_collect) }, // on each theap collect, stats are merged with the parent heap + { MI_GiB, MI_OPTION_UNINIT, MI_OPTION(profile_alloc_interval) },// N bytes (=1 GiB default) between automatic dumps of the `MIMALLOC_PROFILE` heap profiler (use `option_get_size`); 0 disables automatic dumping + { 0, MI_OPTION_UNINIT, MI_OPTION(profile_inuse_interval) }, // N bytes between automatic dumps of the `MIMALLOC_PROFILE` heap profiler whenever in-use bytes grow by that amount (use `option_get_size`); 0 disables (default) + { 0, MI_OPTION_UNINIT, MI_OPTION(profile_time_interval) }, // N seconds between automatic dumps of the `MIMALLOC_PROFILE` heap profiler; 0 disables (default) + { 512*MI_KiB, MI_OPTION_UNINIT, MI_OPTION(profile_sample_rate) }, // sample rate in bytes for the `MIMALLOC_PROFILE` heap profiler (use `option_get_size`) (=512 KiB) + { MI_DEFAULT_STATS_MERGE_THRESHOLD, + MI_OPTION_UNINIT, MI_OPTION(stats_merge_threshold) }, // on a page stats update, merge theap stats into the heap if N bytes were freed/allocated; 0 disables (=512 KiB) + { MI_DEFAULT_ARENA_PURGE_IMMEDIATE_SIZE, + MI_OPTION_UNINIT, MI_OPTION(arena_purge_immediate_size) }, // ranges above this size are purged immediately + }; static void mi_option_init(mi_option_desc_t* desc); +// option value is stored in KiB so it can fit a `long` +// todo: use mi_ssize_t for options? static bool mi_option_has_size_in_kib(mi_option_t option) { return (option == mi_option_reserve_os_memory || option == mi_option_arena_reserve || - option == mi_option_minimal_purge_size || option == mi_option_arena_max_object_size); + option == mi_option_arena_purge_min_size || option == mi_option_arena_purge_immediate_size || + option == mi_option_arena_max_object_size + // || option == mi_option_profile_alloc_interval || option == mi_option_profile_inuse_interval || option == mi_option_profile_sample_rate + ); +} + +// option value is stored in bytes as a `long` is enough. +static bool mi_option_has_size_in_bytes(mi_option_t option) { + return (option == mi_option_stats_merge_threshold || + option == mi_option_profile_alloc_interval || option == mi_option_profile_inuse_interval || option == mi_option_profile_sample_rate); } void _mi_options_init(void) { @@ -243,10 +277,10 @@ mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_ mi_option_t option = (mi_option_t)i; long l = mi_option_get(option); MI_UNUSED(l); // possibly initialize mi_option_desc_t* desc = &mi_options[option]; - _mi_fprintf(out, arg, "option '%s': %ld %s\n", desc->name, desc->value, (mi_option_has_size_in_kib(option) ? "KiB" : "")); + _mi_fprintf(out, arg, "option '%s': %ld %s\n", desc->name, desc->value, (mi_option_has_size_in_kib(option) ? "KiB" : (mi_option_has_size_in_bytes(option) ? "B" : ""))); } - // show build configuration + // show build configuration #if MI_PAGE_META_IS_ALIGNED && MI_PAGE_META_SMALL_IS_ALIGNED _mi_fprintf(out, arg, "free mode : (small) aligned, page size: %zu\n", sizeof(mi_page_t)); #elif MI_PAGE_META_IS_ALIGNED @@ -255,7 +289,7 @@ mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_ _mi_fprintf(out, arg, "free mode : small aligned + pagemap, page size: %zu\n", sizeof(mi_page_t)); #elif MI_FREE_IS_CHECKED _mi_fprintf(out, arg, "free mode : checked, page size: %zu\n", sizeof(mi_page_t)); - #else + #else _mi_fprintf(out, arg, "free mode : pagemap, page size: %zu\n", sizeof(mi_page_t)); #endif #if MI_ENCODE_FREELIST @@ -263,17 +297,17 @@ mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_ #endif #if MI_GUARDED _mi_fprintf(out, arg, "guarded mode: %s, rate=%ld\n", MI_SAMPLE==2 ? "fine-grained" : "enabled", mi_option_get(mi_option_guarded_sample_rate)); - #endif + #endif #if MI_PROFILE _mi_fprintf(out, arg, "profiling : %s\n", MI_SAMPLE==2 ? "fine-grained" : "enabled"); #endif #if MI_TSAN _mi_fprintf(out, arg, "thread santizer enabled\n"); #endif - _mi_fprintf(out, arg, "mem tracking: %s\n", MI_TRACK_TOOL); + _mi_fprintf(out, arg, "mem tracking: %s\n", MI_TRACK_TOOL); _mi_fprintf(out, arg, "debug level : %d\n", MI_DEBUG ); _mi_fprintf(out, arg, "secure level: %d\n", MI_SECURE ); - _mi_fprintf(out, arg, "padding : %s\n", (MI_PADDING==0 ? "none" : (MI_PADDING==1 ? "enabled" : "byte precise"))); + _mi_fprintf(out, arg, "padding : %s\n", (MI_PADDING==0 ? "none" : (MI_PADDING==1 ? "enabled" : "byte precise"))); } mi_decl_export void mi_options_print(void) mi_attr_noexcept { @@ -393,7 +427,7 @@ static void mi_cdecl mi_out_buf(const char* msg, void* arg) { if (start+n >= MI_MAX_DELAY_OUTPUT) { n = MI_MAX_DELAY_OUTPUT-start-1; } - _mi_memcpy(&out_buf[start], msg, n); + mi_memcpy(&out_buf[start], msg, n); } } } @@ -629,14 +663,11 @@ mi_decl_noinline mi_block_t* _mi_block_next_is_corrupted(const mi_page_t* page, _mi_error_message(EFAULT, "corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (uintptr_t)next); return NULL; } - + // -------------------------------------------------------- // Initialize options by checking the environment // -------------------------------------------------------- -// TODO: implement ourselves to reduce dependencies on the C runtime -#include // strtol - static void mi_option_init(mi_option_desc_t* desc) { // Read option value from the environment char s[64 + 1]; @@ -668,25 +699,32 @@ static void mi_option_init(mi_option_desc_t* desc) { desc->init = MI_OPTION_INITIALIZED; } else { - char* end = buf; - errno = 0; - long value = strtol(buf, &end, 10); - if (errno==0 && mi_option_has_size_in_kib(desc->option)) { - // this option is interpreted in KiB to prevent overflow of `long` for large allocations - // (long is 32-bit on 64-bit windows, which allows for 4TiB max.) + const char* end = buf; + long value = 0; + const bool ok = _mi_strntol(buf, sizeof(buf), &end, &value); + if (ok && (mi_option_has_size_in_kib(desc->option) || mi_option_has_size_in_bytes(desc->option))) { + // the size is interpreted in bytes. size_t size = (value < 0 ? 0 : (size_t)value); bool overflow = false; - if (*end == 'K') { end++; } - else if (*end == 'M') { overflow = mi_mul_overflow(size,MI_KiB,&size); end++; } - else if (*end == 'G') { overflow = mi_mul_overflow(size,MI_MiB,&size); end++; } - else if (*end == 'T') { overflow = mi_mul_overflow(size,MI_GiB,&size); end++; } - else { size = (size + MI_KiB - 1) / MI_KiB; } + if (*end == 'K') { overflow = mi_mul_overflow(size,MI_KiB,&size); end++; } + #if MI_SIZE_SIZE >= 4 + else if (*end == 'M') { overflow = mi_mul_overflow(size,MI_MiB,&size); end++; } + else if (*end == 'G') { overflow = mi_mul_overflow(size,MI_GiB,&size); end++; } + #endif + #if MI_SIZE_SIZE >= 8 + else if (*end == 'T') { overflow = mi_mul_overflow(size,MI_GiB * MI_KiB,&size); end++; } + #endif if (end[0] == 'I' && end[1] == 'B') { end += 2; } // KiB, MiB, GiB, TiB else if (*end == 'B') { end++; } // Kb, Mb, Gb, Tb - if (overflow || size > (MI_MAX_ALLOC_SIZE / MI_KiB)) { size = (MI_MAX_ALLOC_SIZE / MI_KiB); } + if (overflow || size > MI_MAX_ALLOC_SIZE) { size = MI_MAX_ALLOC_SIZE; } + if (mi_option_has_size_in_kib(desc->option)) { + // This option is interpreted in KiB to prevent overflow of `long` for large allocations + // (long is 32-bit on 64-bit windows, which allows for 4TiB max.) + size = (size + MI_KiB - 1) / MI_KiB; // no overflow due to check before this + } value = (size > LONG_MAX ? LONG_MAX : (long)size); } - if (errno==0 && *end == 0) { + if (ok && *end == 0) { mi_option_set(desc->option, value); } else { diff --git a/system/lib/mimalloc/src/random.c b/system/lib/mimalloc/src/util/random.c similarity index 92% rename from system/lib/mimalloc/src/random.c rename to system/lib/mimalloc/src/util/random.c index 3c012adf45c4a..1bfad4bc16863 100644 --- a/system/lib/mimalloc/src/random.c +++ b/system/lib/mimalloc/src/util/random.c @@ -1,12 +1,14 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2019-2021, Microsoft Research, Daan Leijen +Copyright (c) 2019-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc/internal.h" #include "mimalloc/prim.h" // _mi_prim_random_buf +#include "mimalloc/util/libc.h" +#include "mimalloc/util/options.h" +#include "mimalloc/util/random.h" /* ---------------------------------------------------------------------------- We use our own PRNG to keep predictable performance of random number generation @@ -95,7 +97,7 @@ static void chacha_init(mi_random_ctx_t* ctx, const uint8_t key[32], uint64_t no // do not _need_ to read 32-bit values as little endian but we do anyways // just for being compatible :-) ctx->output_available = 0; - _mi_memzero(ctx->output,sizeof(ctx->output)); + mi_memzero(ctx->output,sizeof(ctx->output)); for (size_t i = 0; i < 4; i++) { const uint8_t* sigma = (uint8_t*)"expand 32-byte k"; ctx->input[i] = read32(sigma,i); @@ -110,9 +112,9 @@ static void chacha_init(mi_random_ctx_t* ctx, const uint8_t key[32], uint64_t no } static void chacha_split(mi_random_ctx_t* ctx, uint64_t nonce, mi_random_ctx_t* ctx_new) { - _mi_memzero(ctx_new, sizeof(*ctx_new)); + mi_memzero(ctx_new, sizeof(*ctx_new)); ctx_new->weak = ctx->weak; - _mi_memcpy(ctx_new->input, ctx->input, sizeof(ctx_new->input)); + mi_memcpy(ctx_new->input, ctx->input, sizeof(ctx_new->input)); ctx_new->input[12] = 0; ctx_new->input[13] = 0; ctx_new->input[14] = (uint32_t)nonce; @@ -126,21 +128,19 @@ static void chacha_split(mi_random_ctx_t* ctx, uint64_t nonce, mi_random_ctx_t* Random interface -----------------------------------------------------------------------------*/ -#if MI_DEBUG>1 static bool mi_random_is_initialized(mi_random_ctx_t* ctx) { return (ctx != NULL && ctx->input[0] != 0); } -#endif void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* ctx_new) { - mi_assert_internal(mi_random_is_initialized(ctx)); - mi_assert_internal(ctx != ctx_new); + mi_assert_internal(ctx != ctx_new); + if mi_unlikely(!mi_random_is_initialized(ctx)) { _mi_random_init(ctx); } const uintptr_t nonce_rnd = _mi_random_next(ctx); chacha_split(ctx, (uintptr_t)ctx_new ^ nonce_rnd /*nonce*/, ctx_new); } size_t _mi_random_next(mi_random_ctx_t* ctx) { - mi_assert_internal(mi_random_is_initialized(ctx)); + if mi_unlikely(!mi_random_is_initialized(ctx)) { _mi_random_init(ctx); } size_t r; do { #if MI_SIZE_SIZE <= 4 @@ -166,7 +166,7 @@ size_t _mi_os_random_weak(size_t extra_seed) { // and do a few randomization steps size_t max = ((x ^ (x >> 17)) & 0x0F) + 1; for (size_t i = 0; i < max || x==0; i++, x++) { - x = _mi_random_shuffle(x); + x = mi_random_shuffle(x); } mi_assert_internal(x != 0); return x; @@ -181,12 +181,12 @@ static void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) { if (!use_weak) { _mi_warning_message("unable to use secure randomness\n"); } #endif size_t x = _mi_os_random_weak(0); - for (size_t i = 0; i < 32; i+=4, x++) { - x = _mi_random_shuffle(x); + for (size_t i = 0; i < 32; i+=4, x++) { + x = mi_random_shuffle(x); key[i] = (uint8_t)(x); key[i+1] = (uint8_t)(x>>8); key[i+2] = (uint8_t)(x>>16); - key[i+3] = (uint8_t)(x>>24); + key[i+3] = (uint8_t)(x>>24); } ctx->weak = true; } @@ -194,7 +194,7 @@ static void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) { ctx->weak = false; } chacha_init(ctx, key, (uintptr_t)ctx /*nonce*/ ); - _mi_memzero(key, sizeof(key)); + mi_memzero(key, sizeof(key)); } void _mi_random_init(mi_random_ctx_t* ctx) { diff --git a/system/lib/mimalloc/src/util/readme.md b/system/lib/mimalloc/src/util/readme.md new file mode 100644 index 0000000000000..cc38c5b8cd9ad --- /dev/null +++ b/system/lib/mimalloc/src/util/readme.md @@ -0,0 +1,8 @@ +# Utilities + +- [`libc.c`](libc.c): Defines various standard libc functions to reduce +the dependency on libc: string functions, formatting, environment, and other runtime utility functions. +- [`options.c`](options.c): Allocator options, environment-based configuration, and diagnostic output. +- [`random.c`](random.c): Random number generation, seeding, and splitting of random streams. +- [`stats.c`](stats.c): Allocation statistics, aggregation, reporting, and process information. +- [`threadlocal.c`](threadlocal.c): Dynamically allocated thread-local keys and per-thread value storage. diff --git a/system/lib/mimalloc/src/stats.c b/system/lib/mimalloc/src/util/stats.c similarity index 74% rename from system/lib/mimalloc/src/stats.c rename to system/lib/mimalloc/src/util/stats.c index c763bfc3c7b5b..f2a4549024155 100644 --- a/system/lib/mimalloc/src/stats.c +++ b/system/lib/mimalloc/src/util/stats.c @@ -5,13 +5,10 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc-stats.h" -#include "mimalloc/internal.h" -#include "mimalloc/atomic.h" #include "mimalloc/prim.h" // _mi_prim_clock_now, mi_process_info_t -#include "mimalloc/prim-tls.h" - -#include // memset +#include "mimalloc/prim/prim-tls.h" +#include "mimalloc/alloc/page.h" +#include "mimalloc/util/stats.h" #if defined(_MSC_VER) && (_MSC_VER < 1920) #pragma warning(disable:4204) // non-constant aggregate initializer @@ -24,10 +21,10 @@ terms of the MIT license. A copy of the license can be found in the file static void mi_stat_update_mt(mi_stat_count_t* stat, int64_t amount) { if (amount == 0) return; // add atomically - int64_t current = mi_atomic_addi64_relaxed(&stat->current, amount); - mi_atomic_maxi64_relaxed(&stat->peak, current + amount); + int64_t current = mi_atomic_volatile_addi64_relaxed(&stat->current, amount); + mi_atomic_volatile_maxi64_relaxed(&stat->peak, current + amount); if (amount > 0) { - mi_atomic_addi64_relaxed(&stat->total, amount); + mi_atomic_volatile_addi64_relaxed(&stat->total, amount); } } @@ -65,16 +62,16 @@ static void mi_stat_adjust_mt(mi_stat_count_t* stat, int64_t amount) { if (amount == 0) return; // adjust atomically const int64_t peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&stat->peak); - mi_atomic_addi64_relaxed(&stat->current, amount); - const int64_t prev_total = mi_atomic_addi64_relaxed(&stat->total, amount); - if (prev_total == peak) { mi_atomic_addi64_relaxed(&stat->peak, amount); } + mi_atomic_volatile_addi64_relaxed(&stat->current, amount); + const int64_t prev_total = mi_atomic_volatile_addi64_relaxed(&stat->total, amount); + if (prev_total == peak) { mi_atomic_volatile_addi64_relaxed(&stat->peak, amount); } } static void mi_stat_adjust(mi_stat_count_t* stat, int64_t amount) { if (amount == 0) return; stat->current += amount; if (stat->total==stat->peak) { stat->peak += amount; } - stat->total += amount; + stat->total += amount; } void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, uint64_t amount) { @@ -98,23 +95,26 @@ void __mi_stat_adjust_decrease(mi_stat_count_t* stat, uint64_t amount) { // must be thread safe as it is called from stats_merge static void mi_stat_count_add_mt(mi_stat_count_t* stat, const mi_stat_count_t* src) { if (stat==src) return; - mi_atomic_void_addi64_relaxed(&stat->total, &src->total); + mi_atomic_volatile_void_addi64_relaxed(&stat->total, &src->total); const int64_t src_peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&src->peak); const int64_t src_current = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&src->current); - const int64_t prev_current = mi_atomic_addi64_relaxed(&stat->current, src_current); + // avoid atomic operations for common cases + if (src_current == 0 && src_peak == 0) return; + const int64_t prev_current = (src_current == 0 ? mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&stat->current) + : mi_atomic_volatile_addi64_relaxed(&stat->current, src_current)); // Global current plus thread peak approximates new global peak // note: peak scores do not really work across threads. // we used to just add them together but that often overestimates in practice. // similarly, max does not seem to work well. The current approach // by Artem Kharytoniuk (@artem-lunarg) seems to work better, see PR#1112 - // for a longer description. - mi_atomic_maxi64_relaxed(&stat->peak, prev_current + src_peak); + // for a longer description. + mi_atomic_volatile_maxi64_relaxed(&stat->peak, prev_current + src_peak); } static void mi_stat_counter_add_mt(mi_stat_counter_t* stat, const mi_stat_counter_t* src) { if (stat==src) return; - mi_atomic_void_addi64_relaxed(&stat->total, &src->total); + mi_atomic_volatile_void_addi64_relaxed(&stat->total, &src->total); } #define MI_STAT_COUNT(stat) mi_stat_count_add_mt(&stats->stat, &src->stat); @@ -147,20 +147,13 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { Display statistics ----------------------------------------------------------- */ -// unit > 0 : size in binary bytes -// unit == 0: count as decimal -// unit < 0 : count in binary -static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, bool limitwidth) { +static void mi_print_number(int64_t n, mi_output_fun* out, void* arg, bool limitwidth, bool as_bytes, int64_t base) { char buf[32]; _mi_memzero_var(buf); int len = 32; - const char* suffix = (unit <= 0 ? " " : "B"); - const int64_t base = (unit == 0 ? 1000 : 1024); - if (unit>0) n *= unit; - const int64_t pos = (n < 0 ? -n : n); if (pos < base) { - if (n!=1 || suffix[0] != 'B') { // skip printing 1 B for the unit column - _mi_snprintf(buf, len, "%lld %-3s", (long long)n, (n==0 ? "" : suffix)); + if (!(n==1 && as_bytes)) { // skip printing 1 B for the unit column + _mi_snprintf(buf, len, "%lld %-3s", (long long)n, ""); } } else { @@ -172,7 +165,7 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* const long whole = (long)(tens/10); const long frac1 = (long)(tens%10); char unitdesc[8]; - _mi_snprintf(unitdesc, 8, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix); + _mi_snprintf(unitdesc, 8, "%s%s%s", magnitude, (base==1024 ? "i" : ""), (as_bytes ? "B" : "")); _mi_snprintf(buf, len, "%ld.%ld %-3s", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc); } if (limitwidth) { @@ -183,111 +176,119 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* } } - -static void mi_print_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg) { - mi_printf_amount(n,unit,out,arg,true); +static void mi_print_size_narrow(int64_t n, mi_output_fun* out, void* arg) { + mi_print_number(n, out, arg, false, true, 1024); +} +static void mi_print_size(int64_t n, mi_output_fun* out, void* arg) { + mi_print_number(n, out, arg, true, true, 1024); +} +static void mi_print_decimal(int64_t n, mi_output_fun* out, void* arg) { + mi_print_number(n, out, arg, true, false, 1000); } +// static void mi_print_binary(int64_t n, mi_output_fun* out, void* arg) { +// mi_print_number(n, out, arg, true, false, 1024); +// } -static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out, void* arg) { - if (unit==1) _mi_fprintf(out, arg, "%12s"," "); - else mi_print_amount(n,0,out,arg); +static void mi_print_label(const char* label, mi_output_fun* out, void* arg) { + _mi_fprintf(out, arg, " %-12s:", label); } -static void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg, const char* notok ) { - _mi_fprintf(out, arg," %-12s:", msg); - if (unit != 0) { - if (unit > 0) { // as KiB (power of two) - mi_print_amount(stat->peak, unit, out, arg); - mi_print_amount(stat->total, unit, out, arg); - mi_print_amount(stat->current, unit, out, arg); - mi_print_amount(unit, 1, out, arg); - mi_print_count(stat->total, unit, out, arg); - } - else if (unit==-1) { // as K (decimal) - mi_print_amount(stat->peak, -1, out, arg); - mi_print_amount(stat->total, -1, out, arg); - mi_print_amount(stat->current, -1, out, arg); - _mi_fprintf(out, arg, "%24s", ""); - } - else { // as KiB , unit is final count - mi_print_amount(stat->peak, 1, out, arg); - mi_print_amount(stat->total, 1, out, arg); - mi_print_amount(stat->current, -1, out, arg); - _mi_fprintf(out, arg, "%12s", ""); - mi_print_count(-unit, 0, out, arg); - } - if (stat->current != 0) { - _mi_fprintf(out, arg, " "); - _mi_fprintf(out, arg, "%s", (notok == NULL ? "not all freed" : notok)); - _mi_fprintf(out, arg, "\n"); - } - else { - _mi_fprintf(out, arg, " ok\n"); - } - } - else { - mi_print_amount(stat->peak, 0, out, arg); - mi_print_amount(stat->total, 0, out, arg); - mi_print_amount(stat->current, 0, out, arg); - _mi_fprintf(out, arg, "\n"); - } +static void mi_print_newline(mi_output_fun* out, void* arg) { + _mi_fprintf(out, arg, "\n"); } -static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) { - mi_stat_print_ex(stat, msg, unit, out, arg, NULL); +static void mi_print_stat_decimal(const mi_stat_count_t* stat, const char* msg, mi_output_fun* out, void* arg) { + mi_print_label(msg, out, arg); + mi_print_decimal(stat->peak, out, arg); + mi_print_decimal(stat->total, out, arg); + mi_print_decimal(stat->current, out, arg); + mi_print_newline(out, arg); } -#if MI_STATS -static void mi_stat_total_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg) { - _mi_fprintf(out, arg, " %-12s:", msg); - _mi_fprintf(out, arg, "%12s", " "); // no peak - mi_print_amount(stat->total, 1, out, arg); - _mi_fprintf(out, arg, "\n"); +static void mi_print_stat_size(const mi_stat_count_t* stat, const char* msg, mi_output_fun* out, void* arg) { + mi_print_label(msg, out, arg); + mi_print_size(stat->peak, out, arg); + mi_print_size(stat->total, out, arg); + mi_print_size(stat->current, out, arg); + mi_print_newline(out, arg); } -#endif -static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) { - _mi_fprintf(out, arg, " %-12s:", msg); - mi_print_amount(stat->total, 0, out, arg); - _mi_fprintf(out, arg, "\n"); +static void mi_print_stat_counter(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) { + mi_print_label(msg, out, arg); + mi_print_decimal(stat->total, out, arg); + mi_print_newline(out, arg); } -static void mi_stat_counter_print_size(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) { - _mi_fprintf(out, arg, " %-12s:", msg); - mi_print_amount(stat->total, 1, out, arg); - _mi_fprintf(out, arg, "\n"); +static void mi_print_stat_counter_size(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) { + mi_print_label(msg, out, arg); + mi_print_size(stat->total, out, arg); + mi_print_newline(out, arg); } -static void mi_stat_average_print(int64_t count, int64_t total, const char* msg, mi_output_fun* out, void* arg) { +static void mi_print_average(int64_t count, int64_t total, const char* msg, mi_output_fun* out, void* arg) { const int64_t avg_tens = (count == 0 ? 0 : (total*10 / count)); const int64_t avg_whole = avg_tens/10; const int64_t avg_frac1 = avg_tens%10; - _mi_fprintf(out, arg, " %-12s: %5lld.%lld avg\n", msg, avg_whole, avg_frac1); + mi_print_label(msg, out, arg); + _mi_fprintf(out, arg, " %5lld.%lld avg\n", avg_whole, avg_frac1); +} + +static void mi_print_header(const char* name, mi_output_fun* out, void* arg ) { + _mi_fprintf(out, arg, " %-13s %11s %11s %11s\n", name, "peak ", "total ", "current "); } +#if MI_STATS +static void mi_print_empty(mi_output_fun* out, void* arg) { + _mi_fprintf(out, arg, "%12s", " "); +} + +static void mi_print_stat_total_size(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg) { + mi_print_label(msg, out, arg); + mi_print_empty(out, arg); // no peak + mi_print_size(stat->total, out, arg); + mi_print_newline(out, arg); +} + +static void mi_print_check(const mi_stat_count_t* stat, mi_output_fun* out, void* arg, const char* not_ok) { + _mi_fprintf(out, arg, " %s\n", (stat->current==0 ? "ok" : (not_ok == NULL ? "not all freed" : not_ok))); +} -static void mi_print_header(const char* name,mi_output_fun* out, void* arg ) { +static void mi_print_stat_bin(const mi_stat_count_t* stat, const char* msg, int64_t blocksize, int64_t total, mi_output_fun* out, void* arg) { + mi_print_label(msg, out, arg); + mi_print_size(stat->peak * blocksize, out, arg); + mi_print_size(stat->total * blocksize, out, arg); + mi_print_size(stat->current * blocksize, out, arg); + mi_print_size(blocksize, out, arg); + mi_print_decimal(total, out, arg); + mi_print_check(stat, out, arg, NULL); +} + +static void mi_print_header_bins(const char* name, mi_output_fun* out, void* arg ) { _mi_fprintf(out, arg, " %-13s %11s %11s %11s %11s %11s\n", name, "peak ", "total ", "current ", "block ", "total# "); } -#if MI_STATS -static bool mi_stats_print_bins(const mi_stat_count_t* bins, size_t max, mi_output_fun* out, void* arg) { +static bool mi_stats_print_bins(const mi_stat_count_t* bins, const mi_stat_count_t* stat_huge, size_t max, mi_output_fun* out, void* arg) { bool found = false; char buf[64]; for (size_t i = 0; i <= max; i++) { if (bins[i].total > 0) { found = true; - const size_t unit = _mi_bin_size((uint8_t)i); - const char* pagekind = (unit <= MI_SMALL_MAX_OBJ_SIZE ? "S" : - (unit <= MI_MEDIUM_MAX_OBJ_SIZE ? "M" : - (unit <= MI_LARGE_MAX_OBJ_SIZE ? "L" : "H"))); + const size_t blocksize = _mi_bin_size((uint8_t)i); + const char* pagekind = (blocksize <= MI_SMALL_MAX_OBJ_SIZE ? "S" : + (mi_block_size_is_medium(blocksize) ? "M" : + (blocksize <= MI_LARGE_MAX_OBJ_SIZE ? "L" : "H"))); _mi_snprintf(buf, 64, "bin%2s %3lu", pagekind, (long)i); - mi_stat_print(&bins[i], buf, (int64_t)unit, out, arg); + if (i!=MI_BIN_HUGE) { + mi_print_stat_bin(&bins[i], buf, (int64_t)blocksize, bins[i].total, out, arg); + } + else { + mi_print_stat_bin(stat_huge, buf, 1, bins[i].total, out, arg); + } } } if (found) { - _mi_fprintf(out, arg, "\n"); + mi_print_newline(out, arg); } return found; } @@ -343,12 +344,12 @@ mi_decl_export void mi_process_info_print_out(mi_output_fun* out, void* arg) mi_ _mi_fprintf(out, arg, " %-12s: %5zu.%03zu s\n", "elapsed", elapsed/1000, elapsed%1000); _mi_fprintf(out, arg, " %-12s: user: %zu.%03zu s, system: %zu.%03zu s, faults: %zu, peak rss: ", "process", user_time/1000, user_time%1000, sys_time/1000, sys_time%1000, page_faults); - mi_printf_amount((int64_t)peak_rss, 1, out, arg, false); + mi_print_size_narrow((int64_t)peak_rss, out, arg); if (peak_commit > 0) { _mi_fprintf(out, arg, ", peak commit: "); - mi_printf_amount((int64_t)peak_commit, 1, out, arg, false); + mi_print_size_narrow((int64_t)peak_commit, out, arg); } - _mi_fprintf(out, arg, "\n"); + mi_print_newline(out, arg); } void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_attr_noexcept { @@ -362,76 +363,75 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou // and print using that _mi_fprintf(out, arg, "%s %zu\n", name, id); - if (stats->malloc_normal.total + stats->malloc_huge.total != 0) { + if (stats->allocated.total != 0) { #if MI_STATS - mi_print_header("blocks", out, arg); - mi_stats_print_bins(stats->malloc_bins, MI_BIN_HUGE, out, arg); + mi_print_header_bins("blocks", out, arg); + mi_stats_print_bins(stats->malloc_bins, &stats->malloc_huge, MI_BIN_HUGE, out, arg); #endif #if MI_STATS - mi_stat_print(&stats->malloc_normal, "binned", -stats->malloc_normal_count.total, out, arg); - mi_stat_print(&stats->malloc_huge, "huge", -stats->malloc_huge_count.total, out, arg); - mi_stat_count_t total = { 0,0,0 }; - mi_stat_count_add_mt(&total, &stats->malloc_normal); - mi_stat_count_add_mt(&total, &stats->malloc_huge); - mi_stat_print_ex(&total, "total", -(stats->malloc_normal_count.total + stats->malloc_huge_count.total), out, arg, ""); + mi_print_stat_size(&stats->allocated, "allocated", out, arg); + mi_print_stat_total_size(&stats->freed, "freed", out, arg); #if MI_STATS>=2 - mi_stat_total_print(&stats->malloc_requested, "malloc req", out, arg); + mi_print_stat_total_size(&stats->malloc_requested, "requested", out, arg); #else - mi_stat_total_print(&stats->malloc_requested, "malloc req~", out, arg); + mi_print_stat_total_size(&stats->malloc_requested, "requested~", out, arg); #endif - _mi_fprintf(out, arg, "\n"); + mi_print_newline(out, arg); #endif } if (stats->pages.total != 0) { mi_print_header("pages", out, arg); - mi_stat_print_ex(&stats->page_committed, "touched", 1, out, arg, ""); - // mi_stat_print(&stats->segments, "segments", -1, out, arg); - // mi_stat_print(&stats->segments_abandoned, "-abandoned", -1, out, arg); - // mi_stat_print(&stats->segments_cache, "-cached", -1, out, arg); - mi_stat_print(&stats->pages, "pages", 0, out, arg); - mi_stat_print(&stats->pages_abandoned, "abandoned", 0, out, arg); - mi_stat_print(&stats->pages_os_abandoned, "os abandoned", 0, out, arg); - mi_stat_print(&stats->pages_os_allocated, "os allocated", 0, out, arg); - mi_stat_counter_print(&stats->pages_reclaim_on_alloc, "reclaima", out, arg); - mi_stat_counter_print(&stats->pages_reclaim_on_free, "reclaimf", out, arg); - mi_stat_counter_print(&stats->pages_reabandon_full, "reabandon", out, arg); - mi_stat_counter_print(&stats->pages_unabandon_busy_wait, "waits", out, arg); - mi_stat_counter_print(&stats->pages_extended, "extended", out, arg); - mi_stat_counter_print(&stats->pages_retire, "retire", out, arg); - mi_stat_counter_print(&stats->pages_stat_updates, "stat updates", out, arg); - mi_stat_average_print(stats->pages_stat_updates.total, stats->pages_stat_update_count.total, "stat avg", out, arg); - mi_stat_average_print(stats->page_searches_count.total, stats->page_searches.total, "searches", out, arg); - mi_stat_counter_print(&stats->profile_samples, "prof samples", out, arg); - _mi_fprintf(out, arg, "\n"); + mi_print_stat_size(&stats->page_committed, "touched", out, arg); + // mi_print_stat(&stats->segments, "segments", -1, out, arg); + // mi_print_stat(&stats->segments_abandoned, "-abandoned", -1, out, arg); + // mi_print_stat(&stats->segments_cache, "-cached", -1, out, arg); + mi_print_stat_decimal(&stats->pages, "pages", out, arg); + mi_print_stat_decimal(&stats->pages_abandoned, "abandoned", out, arg); + mi_print_stat_decimal(&stats->pages_os_abandoned, "os abandoned", out, arg); + mi_print_stat_decimal(&stats->pages_os_allocated, "os allocated", out, arg); + + mi_print_stat_counter(&stats->pages_reclaim_on_alloc, "reclaima", out, arg); + mi_print_stat_counter(&stats->pages_reclaim_on_free, "reclaimf", out, arg); + mi_print_stat_counter(&stats->pages_reabandon_full, "reabandon", out, arg); + mi_print_stat_counter(&stats->pages_unabandon_busy_wait, "unabdn retry", out, arg); + mi_print_stat_counter(&stats->pages_unown_cas_retry, "unown retry", out, arg); + mi_print_stat_counter(&stats->pages_extended, "extended", out, arg); + mi_print_stat_counter(&stats->pages_retire, "retire", out, arg); + mi_print_stat_counter(&stats->pages_stat_updates, "stat updates", out, arg); + mi_print_average(stats->pages_stat_updates.total, stats->pages_stat_update_count.total, "stat avg", out, arg); + mi_print_average(stats->page_searches_count.total, stats->page_searches.total, "searches", out, arg); + mi_print_stat_counter(&stats->profile_samples, "prof samples", out, arg); + mi_print_newline(out, arg); } if (stats->arena_count.total > 0) { mi_print_header("arenas", out, arg); - mi_stat_print_ex(&stats->reserved, "reserved", 1, out, arg, ""); - mi_stat_print_ex(&stats->committed, "committed", 1, out, arg, ""); - mi_stat_counter_print_size(&stats->reset, "reset", out, arg); - mi_stat_counter_print_size(&stats->purged, "purged", out, arg); - - mi_stat_counter_print(&stats->arena_count, "arenas", out, arg); - mi_stat_counter_print(&stats->arena_rollback_count, "rollback", out, arg); - mi_stat_counter_print(&stats->mmap_calls, "mmaps", out, arg); - mi_stat_counter_print(&stats->commit_calls, "commits", out, arg); - mi_stat_counter_print(&stats->reset_calls, "resets", out, arg); - mi_stat_counter_print(&stats->purge_calls, "purges", out, arg); - mi_stat_counter_print(&stats->malloc_guarded_count, "guarded", out, arg); - mi_stat_print_ex(&stats->theaps, "theaps", 0, out, arg, ""); - mi_stat_print_ex(&stats->heaps, "heaps", 0, out, arg, ""); - mi_stat_counter_print(&stats->heaps_delete_wait, "heap waits", out, arg); - _mi_fprintf(out, arg, "\n"); + mi_print_stat_size(&stats->reserved, "reserved", out, arg); + mi_print_stat_size(&stats->committed, "committed", out, arg); + mi_print_stat_counter_size(&stats->reset, "reset", out, arg); + mi_print_stat_counter_size(&stats->purged, "purged", out, arg); + + mi_print_stat_counter(&stats->arena_count, "arenas", out, arg); + mi_print_stat_counter(&stats->arena_rollback_count, "rollback", out, arg); + mi_print_stat_counter(&stats->mmap_calls, "mmaps", out, arg); + mi_print_stat_counter(&stats->commit_calls, "commits", out, arg); + mi_print_stat_counter(&stats->reset_calls, "resets", out, arg); + mi_print_stat_counter(&stats->purge_calls, "purges", out, arg); + mi_print_stat_counter(&stats->malloc_guarded_count, "guarded", out, arg); + mi_print_stat_decimal(&stats->theaps, "theaps", out, arg); + mi_print_stat_decimal(&stats->heaps, "heaps", out, arg); + mi_print_stat_counter(&stats->heaps_delete_wait, "heap waits", out, arg); + mi_print_newline(out, arg); } mi_print_header("process", out, arg); - mi_stat_print_ex(&stats->threads, "threads", 0, out, arg, ""); - _mi_fprintf(out, arg, " %-12s: %5i\n", "numa nodes", _mi_os_numa_node_count()); + mi_print_stat_decimal(&stats->threads, "threads", out, arg); + mi_stat_counter_t numa_count = { _mi_os_numa_node_count() }; + mi_print_stat_counter(&numa_count, "numa nodes", out, arg); mi_process_info_print_out(out, arg); - - _mi_fprintf(out, arg, "\n"); + + mi_print_newline(out, arg); } @@ -452,12 +452,12 @@ void _mi_stats_merge_into(mi_stats_t* to, mi_stats_t* from) { mi_assert_internal(to != NULL && from != NULL); if (to == from) return; mi_stats_add(to, from); - mi_stats_init(from); // zero field and keep the header + mi_stats_init(from); // zero field and keep the header } static const mi_stats_t* mi_stats_merge_theap_to_heap(mi_theap_t* theap) mi_attr_noexcept { mi_stats_t* stats = &theap->stats; - mi_stats_t* heap_stats = &_mi_theap_heap(theap)->stats; + mi_stats_t* heap_stats = &mi_theap_heap(theap)->stats; _mi_stats_merge_into( heap_stats, stats ); return heap_stats; } @@ -537,7 +537,7 @@ void mi_stats_print(void* out) mi_attr_noexcept { void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept { mi_theap_t* theap = _mi_theap_default(); if (theap==NULL || !mi_theap_is_initialized(theap)) return; - _mi_stats_print("heap", _mi_theap_heap(theap)->heap_seq, &theap->stats, out, arg); + _mi_stats_print("heap", mi_theap_heap(theap)->heap_seq, &theap->stats, out, arg); mi_stats_merge_theap_to_heap(_mi_theap_default()); } @@ -581,7 +581,7 @@ mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, s const int64_t peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.peak)); pinfo.current_commit = (current < 0 ? 0 : (current < PTRDIFF_MAX ? (size_t)current : PTRDIFF_MAX)); pinfo.peak_commit = (peak < 0 ? 0 : (peak < PTRDIFF_MAX ? (size_t)peak : PTRDIFF_MAX)); - } + } } pinfo.current_rss = pinfo.current_commit; pinfo.peak_rss = pinfo.peak_commit; @@ -618,7 +618,7 @@ size_t mi_stats_get_bin_size(size_t bin) mi_attr_noexcept { static bool mi_stats_copy(mi_stats_t* stats_to, const mi_stats_t* stats_from) mi_attr_noexcept { if (stats_to == NULL || stats_to->size != sizeof(mi_stats_t) || stats_to->version != MI_STAT_VERSION) return false; if (stats_from == NULL || stats_from->size != stats_to->size) return false; - _mi_memcpy(stats_to, stats_from, stats_to->size); + mi_memcpy(stats_to, stats_from, stats_to->size); return true; } @@ -706,7 +706,7 @@ static void mi_json_buf_print(mi_json_buf_t* hbuf, const char* msg) { static void mi_json_buf_print_count_bin(mi_json_buf_t* hbuf, const char* prefix, const mi_stat_count_t* stat, size_t bin, bool add_comma) { const size_t binsize = mi_stats_get_bin_size(bin); const size_t pagesize = (binsize <= MI_SMALL_MAX_OBJ_SIZE ? MI_SMALL_PAGE_SIZE : - (binsize <= MI_MEDIUM_MAX_OBJ_SIZE ? MI_MEDIUM_PAGE_SIZE : + (mi_block_size_is_medium(binsize) ? MI_MEDIUM_PAGE_SIZE : (binsize <= MI_LARGE_MAX_OBJ_SIZE ? MI_LARGE_PAGE_SIZE : 0))); char buf[128]; _mi_snprintf(buf, 128, "%s{ \"total\": %lld, \"peak\": %lld, \"current\": %lld, \"block_size\": %zu, \"page_size\": %zu }%s\n", prefix, stat->total, stat->peak, stat->current, binsize, pagesize, (add_comma ? "," : "")); @@ -767,7 +767,7 @@ static char* mi_stats_get_json_from(const mi_stats_t* stats, size_t output_size, if (stats==NULL || stats->size!=sizeof(mi_stats_t) || stats->version!=MI_STAT_VERSION) return NULL; mi_json_buf_t hbuf = { NULL, 0, 0, true }; if (output_size > 0 && output_buf != NULL) { - _mi_memzero(output_buf, output_size); + mi_memzero(output_buf, output_size); hbuf.buf = output_buf; hbuf.size = output_size; hbuf.can_realloc = false; @@ -805,7 +805,7 @@ static char* mi_stats_get_json_from(const mi_stats_t* stats, size_t output_size, #define MI_STAT_COUNTER(stat) mi_json_buf_print_counter_value(&hbuf, #stat, &stats->stat); MI_STAT_FIELDS() - + #undef MI_STAT_COUNT #undef MI_STAT_COUNTER diff --git a/system/lib/mimalloc/src/threadlocal.c b/system/lib/mimalloc/src/util/threadlocal.c similarity index 97% rename from system/lib/mimalloc/src/threadlocal.c rename to system/lib/mimalloc/src/util/threadlocal.c index 6f280d09353c9..3a8a8fdc463cc 100644 --- a/system/lib/mimalloc/src/threadlocal.c +++ b/system/lib/mimalloc/src/util/threadlocal.c @@ -12,8 +12,10 @@ terms of the MIT license. A copy of the license can be found in the file -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc/internal.h" -#include "mimalloc/prim.h" +#include "mimalloc/prim/prim-tls.h" +#include "mimalloc/util/libc.h" +#include "mimalloc/util/options.h" +#include "mimalloc/util/threadlocal.h" /* ----------------------------------------------------------- Each thread can have (a dynamically expanding) array of @@ -35,7 +37,7 @@ static mi_thread_locals_t mi_thread_locals_empty = mi_init_struct_zero; /* ----------------------------------------------------------- - We have 2 thread local variable which we implement with either + We have 2 thread local variables which we implement with either a C thread local declaration or using pthread keys. - mi_thread_locals: points to an array of thread locals for most keys - mi_slot_fast: a single dedicated thread local for slightly faster access. (used for the main heap's theap) @@ -216,7 +218,7 @@ void _mi_thread_locals_thread_done(void) { /* ----------------------------------------------------------- Create and free fresh TLS key's ----------------------------------------------------------- */ -#include "bitmap.h" +#include "../arena/bitmap.h" static mi_lock_t mi_thread_locals_lock; // we need a lock in order to re-allocate the slot bits static mi_bitmap_t* mi_thread_locals_free; // reuse an arena bitmap to track which slots were assigned (1=free, 0=in-use) @@ -270,12 +272,12 @@ static bool mi_thread_local_create_expand(void) { // mi_bitmap_t* newslots = (mi_bitmap_t*)mi_zalloc_aligned(newsize, MI_BCHUNK_SIZE); mi_memid_t memid; mi_bitmap_t* newslots = (mi_bitmap_t*)_mi_meta_zalloc_aligned(_mi_subproc_main(), newsize, MI_BCHUNK_SIZE, &memid); // always allocate thread locals in the main subprocess - mi_assert_internal(_mi_is_aligned(newslots,MI_BCHUNK_SIZE)); + mi_assert_internal(mi_is_aligned(newslots,MI_BCHUNK_SIZE)); if (newslots==NULL) { return false; } if (slots!=NULL) { // copy over the previous bitmap const size_t oldsize = mi_bitmap_size(oldcount,NULL); - _mi_memcpy_aligned(newslots, slots, oldsize); + mi_memcpy_aligned(newslots, slots, oldsize); _mi_meta_free(_mi_subproc_main(), slots, mi_thread_locals_memid); } mi_bitmap_init(newslots, newcount, true /* pretend already zero'd so we do not zero out the copied old entries */); @@ -313,4 +315,3 @@ void _mi_thread_local_free(mi_thread_local_t key) { } } } - diff --git a/test/test_other.py b/test/test_other.py index f4162deb73fad..8abe3781eab2b 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -7835,10 +7835,10 @@ def test_dlmalloc_modes(self): 'O2': (['-O2'], 137000), 'emmalloc': (['-sMALLOC=emmalloc'], 185000), 'dlmalloc': (['-sMALLOC=dlmalloc'], 191000), - 'mimalloc': (['-sMALLOC=mimalloc'], 260000), + 'mimalloc': (['-sMALLOC=mimalloc'], 272000), 'emmalloc_O2': (['-sMALLOC=emmalloc', '-O2'], 131000), 'dlmalloc_O2': (['-sMALLOC=dlmalloc', '-O2'], 137000), - 'mimalloc_O2': (['-sMALLOC=mimalloc', '-O2'], 200000), + 'mimalloc_O2': (['-sMALLOC=mimalloc', '-O2'], 214000), }) # This test verifies the output code size of the different -sMALLOC= modes. def test_malloc_size(self, args, max_size): @@ -15495,6 +15495,7 @@ def test_mimalloc_headers(self): int main() { mi_option_enable(mi_option_verbose); + mi_options_print(); return 0; } ''' diff --git a/tools/system_libs.py b/tools/system_libs.py index b74670831351e..3b609c7c632f1 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -1910,11 +1910,16 @@ class libmimalloc(MTLibrary): # allocator underneath it. src_dir = 'system/lib/' src_files = glob_in_path( - path='system/lib/mimalloc/src', + path='system/lib/mimalloc/src/alloc', glob_pattern='*.c', - # mimalloc includes some files at the source level, so exclude them here. - excludes={'alloc-override.c', 'free.c', 'page-queue.c', 'static.c'}, + # included from alloc.c, so exclude them here. + excludes={'alloc-override.c', 'free.c'}, ) + src_files += glob_in_path('system/lib/mimalloc/src/arena', '*.c') + src_files += glob_in_path('system/lib/mimalloc/src/heap', '*.c') + src_files += glob_in_path('system/lib/mimalloc/src/profile', '*.c') + src_files += glob_in_path('system/lib/mimalloc/src/util', '*.c') + src_files += [utils.path_from_root('system/lib/mimalloc/src/prim/os.c')] src_files += [utils.path_from_root('system/lib/mimalloc/src/prim/prim.c')] src_files += [utils.path_from_root('system/lib/mimalloc/src/prim/prim-tls.c')] src_files += [utils.path_from_root('system/lib/emmalloc.c')]