chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2018-2025 Microsoft Corporation, Daan Leijen
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,66 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2020 Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#pragma once
|
||||
#ifndef MIMALLOC_NEW_DELETE_H
|
||||
#define MIMALLOC_NEW_DELETE_H
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// This header provides convenient overrides for the new and
|
||||
// delete operations in C++.
|
||||
//
|
||||
// This header should be included in only one source file!
|
||||
//
|
||||
// On Windows, or when linking dynamically with mimalloc, these
|
||||
// can be more performant than the standard new-delete operations.
|
||||
// See <https://en.cppreference.com/w/cpp/memory/new/operator_new>
|
||||
// ---------------------------------------------------------------------------
|
||||
#if defined(__cplusplus)
|
||||
#include <new>
|
||||
#include <mimalloc.h>
|
||||
|
||||
#if defined(_MSC_VER) && defined(_Ret_notnull_) && defined(_Post_writable_byte_size_)
|
||||
// stay consistent with VCRT definitions
|
||||
#define mi_decl_new(n) mi_decl_nodiscard mi_decl_restrict _Ret_notnull_ _Post_writable_byte_size_(n)
|
||||
#define mi_decl_new_nothrow(n) mi_decl_nodiscard mi_decl_restrict _Ret_maybenull_ _Success_(return != NULL) _Post_writable_byte_size_(n)
|
||||
#else
|
||||
#define mi_decl_new(n) mi_decl_nodiscard mi_decl_restrict
|
||||
#define mi_decl_new_nothrow(n) mi_decl_nodiscard mi_decl_restrict
|
||||
#endif
|
||||
|
||||
void operator delete(void* p) noexcept { mi_free(p); };
|
||||
void operator delete[](void* p) noexcept { mi_free(p); };
|
||||
|
||||
void operator delete (void* p, const std::nothrow_t&) noexcept { mi_free(p); }
|
||||
void operator delete[](void* p, const std::nothrow_t&) noexcept { mi_free(p); }
|
||||
|
||||
mi_decl_new(n) void* operator new(std::size_t n) noexcept(false) { return mi_new(n); }
|
||||
mi_decl_new(n) void* operator new[](std::size_t n) noexcept(false) { return mi_new(n); }
|
||||
|
||||
mi_decl_new_nothrow(n) void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_new_nothrow(n); }
|
||||
mi_decl_new_nothrow(n) void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_new_nothrow(n); }
|
||||
|
||||
#if (__cplusplus >= 201402L || _MSC_VER >= 1916)
|
||||
void operator delete (void* p, std::size_t n) noexcept { mi_free_size(p,n); };
|
||||
void operator delete[](void* p, std::size_t n) noexcept { mi_free_size(p,n); };
|
||||
#endif
|
||||
|
||||
#if (__cplusplus > 201402L || defined(__cpp_aligned_new))
|
||||
void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
|
||||
void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
|
||||
void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
|
||||
void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
|
||||
void operator delete (void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
|
||||
void operator delete[](void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
|
||||
|
||||
void* operator new (std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
|
||||
void* operator new[](std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
|
||||
void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast<size_t>(al)); }
|
||||
void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast<size_t>(al)); }
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#endif // MIMALLOC_NEW_DELETE_H
|
||||
@@ -0,0 +1,68 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2020 Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#pragma once
|
||||
#ifndef MIMALLOC_OVERRIDE_H
|
||||
#define MIMALLOC_OVERRIDE_H
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
This header can be used to statically redirect malloc/free and new/delete
|
||||
to the mimalloc variants. This can be useful if one can include this file on
|
||||
each source file in a project (but be careful when using external code to
|
||||
not accidentally mix pointers from different allocators).
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
#include <mimalloc.h>
|
||||
|
||||
// Standard C allocation
|
||||
#define malloc(n) mi_malloc(n)
|
||||
#define calloc(c,n) mi_calloc(c,n)
|
||||
#define realloc(p,n) mi_realloc(p,n)
|
||||
#define free(p) mi_free(p)
|
||||
|
||||
#define strdup(s) mi_strdup(s)
|
||||
#define strndup(s,n) mi_strndup(s,n)
|
||||
#define realpath(f,n) mi_realpath(f,n)
|
||||
|
||||
// Microsoft extensions
|
||||
#define _expand(p,n) mi_expand(p,n)
|
||||
#define _msize(p) mi_usable_size(p)
|
||||
#define _recalloc(p,c,n) mi_recalloc(p,c,n)
|
||||
|
||||
#define _strdup(s) mi_strdup(s)
|
||||
#define _strndup(s,n) mi_strndup(s,n)
|
||||
#define _wcsdup(s) mi_wcsdup(s)
|
||||
#define _mbsdup(s) mi_mbsdup(s)
|
||||
#define _dupenv_s(buf,n,nm) mi_dupenv_s(buf,n,nm)
|
||||
#define _wdupenv_s(buf,n,nm) mi_wdupenv_s(buf,n,nm)
|
||||
|
||||
// Various Posix and Unix variants
|
||||
#define reallocf(p,n) mi_reallocf(p,n)
|
||||
#define malloc_size(p) mi_usable_size(p)
|
||||
#define malloc_usable_size(p) mi_usable_size(p)
|
||||
#define malloc_good_size(n) mi_malloc_good_size(n)
|
||||
#define cfree(p) mi_free(p)
|
||||
|
||||
#define valloc(n) mi_valloc(n)
|
||||
#define pvalloc(n) mi_pvalloc(n)
|
||||
#define reallocarray(p,c,n) mi_reallocarray(p,c,n)
|
||||
#define reallocarr(ptrp,c,n) mi_reallocarr(ptrp,c,n)
|
||||
#define memalign(a,n) mi_memalign(a,n)
|
||||
#define aligned_alloc(a,n) mi_aligned_alloc(a,n)
|
||||
#define posix_memalign(p,a,n) mi_posix_memalign(p,a,n)
|
||||
#define _posix_memalign(p,a,n) mi_posix_memalign(p,a,n)
|
||||
|
||||
// Microsoft aligned variants
|
||||
#define _aligned_malloc(n,a) mi_malloc_aligned(n,a)
|
||||
#define _aligned_realloc(p,n,a) mi_realloc_aligned(p,n,a)
|
||||
#define _aligned_recalloc(p,c,n,a) mi_aligned_recalloc(p,c,n,a)
|
||||
#define _aligned_msize(p,a,o) mi_usable_size(p)
|
||||
#define _aligned_free(p) mi_free(p)
|
||||
#define _aligned_offset_malloc(n,a,o) mi_malloc_aligned_at(n,a,o)
|
||||
#define _aligned_offset_realloc(p,n,a,o) mi_realloc_aligned_at(p,n,a,o)
|
||||
#define _aligned_offset_recalloc(p,c,n,a,o) mi_recalloc_aligned_at(p,c,n,a,o)
|
||||
|
||||
#endif // MIMALLOC_OVERRIDE_H
|
||||
@@ -0,0 +1,168 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2024-2025, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#pragma once
|
||||
#ifndef MIMALLOC_STATS_H
|
||||
#define MIMALLOC_STATS_H
|
||||
|
||||
#include <mimalloc.h>
|
||||
#include <string.h> // memset
|
||||
#include <stdint.h> // int64_t
|
||||
|
||||
#define MI_STAT_VERSION 5 // increased on every backward incompatible change
|
||||
|
||||
// alignment for atomic fields
|
||||
#if defined(_MSC_VER)
|
||||
#define mi_decl_align(a) __declspec(align(a))
|
||||
#elif defined(__GNUC__)
|
||||
#define mi_decl_align(a) __attribute__((aligned(a)))
|
||||
#elif __cplusplus >= 201103L
|
||||
#define mi_decl_align(a) alignas(a)
|
||||
#else
|
||||
#define mi_decl_align(a)
|
||||
#endif
|
||||
|
||||
|
||||
// count allocation over time
|
||||
typedef struct mi_stat_count_s {
|
||||
int64_t total; // total allocated
|
||||
int64_t peak; // peak allocation
|
||||
int64_t current; // current allocation
|
||||
} mi_stat_count_t;
|
||||
|
||||
// counters only increase
|
||||
typedef struct mi_stat_counter_s {
|
||||
int64_t total; // total count
|
||||
} mi_stat_counter_t;
|
||||
|
||||
#define MI_STAT_FIELDS() \
|
||||
MI_STAT_COUNT(pages) /* count of mimalloc pages */ \
|
||||
MI_STAT_COUNT(reserved) /* reserved memory bytes */ \
|
||||
MI_STAT_COUNT(committed) /* committed bytes */ \
|
||||
MI_STAT_COUNTER(reset) /* reset bytes */ \
|
||||
MI_STAT_COUNTER(purged) /* purged bytes */ \
|
||||
MI_STAT_COUNT(page_committed) /* committed memory inside pages */ \
|
||||
MI_STAT_COUNT(pages_abandoned) /* abandoned pages count */ \
|
||||
MI_STAT_COUNT(threads) /* number of threads */ \
|
||||
MI_STAT_COUNT(malloc_normal) /* allocated bytes <= MI_LARGE_OBJ_SIZE_MAX */ \
|
||||
MI_STAT_COUNT(malloc_huge) /* allocated bytes in huge pages */ \
|
||||
MI_STAT_COUNT(malloc_requested) /* malloc requested bytes */ \
|
||||
\
|
||||
MI_STAT_COUNTER(mmap_calls) \
|
||||
MI_STAT_COUNTER(commit_calls) \
|
||||
MI_STAT_COUNTER(reset_calls) \
|
||||
MI_STAT_COUNTER(purge_calls) \
|
||||
MI_STAT_COUNTER(arena_count) /* number of memory arena's */ \
|
||||
MI_STAT_COUNTER(malloc_normal_count) /* number of blocks <= MI_LARGE_OBJ_SIZE_MAX */ \
|
||||
MI_STAT_COUNTER(malloc_huge_count) /* number of huge bloks */ \
|
||||
MI_STAT_COUNTER(malloc_guarded_count) /* number of allocations with guard pages */ \
|
||||
\
|
||||
/* internal statistics */ \
|
||||
MI_STAT_COUNTER(arena_rollback_count) \
|
||||
MI_STAT_COUNTER(arena_purges) \
|
||||
MI_STAT_COUNTER(pages_extended) /* number of page extensions */ \
|
||||
MI_STAT_COUNTER(pages_retire) /* number of pages that are retired */ \
|
||||
MI_STAT_COUNTER(page_searches) /* total pages searched for a fresh page */ \
|
||||
MI_STAT_COUNTER(page_searches_count) /* searched count for a fresh page */ \
|
||||
/* only on v1 and v2 */ \
|
||||
MI_STAT_COUNT(segments) \
|
||||
MI_STAT_COUNT(segments_abandoned) \
|
||||
MI_STAT_COUNT(segments_cache) \
|
||||
MI_STAT_COUNT(_segments_reserved) \
|
||||
/* only on v3 */ \
|
||||
MI_STAT_COUNT(heaps) \
|
||||
MI_STAT_COUNT(theaps) \
|
||||
MI_STAT_COUNTER(pages_reclaim_on_alloc) \
|
||||
MI_STAT_COUNTER(pages_reclaim_on_free) \
|
||||
MI_STAT_COUNTER(pages_reabandon_full) \
|
||||
MI_STAT_COUNTER(pages_unabandon_busy_wait) \
|
||||
MI_STAT_COUNTER(heaps_delete_wait)
|
||||
|
||||
// Size bins for chunks
|
||||
typedef enum mi_chunkbin_e {
|
||||
MI_CBIN_SMALL, // slice_count == 1
|
||||
MI_CBIN_OTHER, // slice_count: any other from the other bins, and 1 <= slice_count <= MI_BCHUNK_BITS
|
||||
MI_CBIN_MEDIUM, // slice_count == 8
|
||||
MI_CBIN_LARGE, // slice_count == MI_SIZE_BITS (only used if MI_ENABLE_LARGE_PAGES is 1)
|
||||
MI_CBIN_HUGE, // slice_count > MI_BCHUNK_BITS
|
||||
MI_CBIN_NONE, // no bin assigned yet (the chunk is completely free)
|
||||
MI_CBIN_COUNT
|
||||
} mi_chunkbin_t;
|
||||
|
||||
|
||||
// Define the statistics structure
|
||||
#define MI_BIN_HUGE (73U) // see types.h
|
||||
#define MI_STAT_COUNT(stat) mi_stat_count_t stat;
|
||||
#define MI_STAT_COUNTER(stat) mi_stat_counter_t stat;
|
||||
|
||||
typedef struct mi_stats_s
|
||||
{
|
||||
size_t size; // size of the mi_stats_t structure
|
||||
size_t version;
|
||||
|
||||
mi_decl_align(8) MI_STAT_FIELDS()
|
||||
|
||||
// future extension
|
||||
mi_stat_count_t _stat_reserved[4];
|
||||
mi_stat_counter_t _stat_counter_reserved[4];
|
||||
|
||||
// size segregated statistics
|
||||
mi_stat_count_t malloc_bins[MI_BIN_HUGE+1]; // allocation per size bin
|
||||
mi_stat_count_t page_bins[MI_BIN_HUGE+1]; // pages allocated per size bin
|
||||
mi_stat_count_t chunk_bins[MI_CBIN_COUNT]; // chunks per page sizes
|
||||
} mi_stats_t;
|
||||
|
||||
#undef MI_STAT_COUNT
|
||||
#undef MI_STAT_COUNTER
|
||||
|
||||
// Initialization
|
||||
static inline void mi_stats_header_init(mi_stats_t* stats) {
|
||||
stats->size = sizeof(*stats);
|
||||
stats->version = MI_STAT_VERSION;
|
||||
}
|
||||
static inline void mi_stats_init(mi_stats_t* stats) {
|
||||
memset(stats,0,sizeof(*stats));
|
||||
mi_stats_header_init(stats);
|
||||
}
|
||||
|
||||
#define mi_stats_t_decl(name) mi_stats_t name; mi_stats_init(&name);
|
||||
|
||||
// Exported definitions
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
// stats from a heap
|
||||
mi_decl_export bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats) mi_attr_noexcept;
|
||||
mi_decl_export char* mi_heap_stats_get_json(mi_heap_t* heap, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL
|
||||
mi_decl_export void mi_heap_stats_print_out(mi_heap_t* heap, mi_output_fun* out, void* arg) mi_attr_noexcept;
|
||||
|
||||
// stats from a subprocess and its heaps aggregated
|
||||
mi_decl_export bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept;
|
||||
mi_decl_export char* mi_subproc_stats_get_json(mi_subproc_id_t subproc_id, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL
|
||||
mi_decl_export void mi_subproc_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg) mi_attr_noexcept;
|
||||
// print subprocess and all its heap stats segregated
|
||||
mi_decl_export void mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg) mi_attr_noexcept;
|
||||
|
||||
// stats aggregated for the current subprocess and all its heaps.
|
||||
mi_decl_export bool mi_stats_get(mi_stats_t* stats) mi_attr_noexcept;
|
||||
mi_decl_export char* mi_stats_get_json(size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL
|
||||
mi_decl_export void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
|
||||
|
||||
// add the stats of the heap to the subprocess and clear the heap stats
|
||||
mi_decl_export void mi_heap_stats_merge_to_subproc(mi_heap_t* heap);
|
||||
|
||||
// stats from the subprocess without aggregating its heaps
|
||||
mi_decl_export bool mi_subproc_stats_get_exclusive(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept;
|
||||
|
||||
mi_decl_export char* mi_stats_as_json(mi_stats_t* stats, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL
|
||||
mi_decl_export size_t mi_stats_get_bin_size(size_t bin) mi_attr_noexcept;
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // MIMALLOC_STATS_H
|
||||
@@ -0,0 +1,706 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#pragma once
|
||||
#ifndef MIMALLOC_H
|
||||
#define MIMALLOC_H
|
||||
|
||||
#define MI_MALLOC_VERSION 30302 // major + 2 digits minor + 2 digits patch
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Compiler specific attributes
|
||||
// ------------------------------------------------------
|
||||
|
||||
#ifdef __cplusplus
|
||||
#if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11
|
||||
#define mi_attr_noexcept noexcept
|
||||
#else
|
||||
#define mi_attr_noexcept throw()
|
||||
#endif
|
||||
#else
|
||||
#define mi_attr_noexcept
|
||||
#endif
|
||||
|
||||
#if defined(__cplusplus) && (__cplusplus >= 201703)
|
||||
#define mi_decl_nodiscard [[nodiscard]]
|
||||
#elif (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) // includes clang, icc, and clang-cl
|
||||
#define mi_decl_nodiscard __attribute__((warn_unused_result))
|
||||
#elif defined(_HAS_NODISCARD)
|
||||
#define mi_decl_nodiscard _NODISCARD
|
||||
#elif (_MSC_VER >= 1700)
|
||||
#define mi_decl_nodiscard _Check_return_
|
||||
#else
|
||||
#define mi_decl_nodiscard
|
||||
#endif
|
||||
|
||||
#if defined(_MSC_VER) || defined(__MINGW32__)
|
||||
#if !defined(MI_SHARED_LIB)
|
||||
#define mi_decl_export
|
||||
#elif defined(MI_SHARED_LIB_EXPORT)
|
||||
#define mi_decl_export __declspec(dllexport)
|
||||
#else
|
||||
#define mi_decl_export __declspec(dllimport)
|
||||
#endif
|
||||
#if defined(__MINGW32__)
|
||||
#define mi_decl_restrict
|
||||
#define mi_attr_malloc __attribute__((malloc))
|
||||
#else
|
||||
#if (_MSC_VER >= 1900) && !defined(__EDG__)
|
||||
#define mi_decl_restrict __declspec(allocator) __declspec(restrict)
|
||||
#else
|
||||
#define mi_decl_restrict __declspec(restrict)
|
||||
#endif
|
||||
#define mi_attr_malloc
|
||||
#endif
|
||||
#define mi_cdecl __cdecl
|
||||
#define mi_attr_alloc_size(s)
|
||||
#define mi_attr_alloc_size2(s1,s2)
|
||||
#define mi_attr_alloc_align(p)
|
||||
#elif defined(__GNUC__) // includes clang and icc
|
||||
#if defined(MI_SHARED_LIB) && defined(MI_SHARED_LIB_EXPORT)
|
||||
#define mi_decl_export __attribute__((visibility("default")))
|
||||
#else
|
||||
#define mi_decl_export
|
||||
#endif
|
||||
#define mi_cdecl // leads to warnings... __attribute__((cdecl))
|
||||
#define mi_decl_restrict
|
||||
#define mi_attr_malloc __attribute__((malloc))
|
||||
#if (defined(__clang_major__) && (__clang_major__ < 4)) || (__GNUC__ < 5)
|
||||
#define mi_attr_alloc_size(s)
|
||||
#define mi_attr_alloc_size2(s1,s2)
|
||||
#define mi_attr_alloc_align(p)
|
||||
#elif defined(__INTEL_COMPILER)
|
||||
#define mi_attr_alloc_size(s) __attribute__((alloc_size(s)))
|
||||
#define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2)))
|
||||
#define mi_attr_alloc_align(p)
|
||||
#else
|
||||
#define mi_attr_alloc_size(s) __attribute__((alloc_size(s)))
|
||||
#define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2)))
|
||||
#define mi_attr_alloc_align(p) __attribute__((alloc_align(p)))
|
||||
#endif
|
||||
#else
|
||||
#define mi_cdecl
|
||||
#define mi_decl_export
|
||||
#define mi_decl_restrict
|
||||
#define mi_attr_malloc
|
||||
#define mi_attr_alloc_size(s)
|
||||
#define mi_attr_alloc_size2(s1,s2)
|
||||
#define mi_attr_alloc_align(p)
|
||||
#endif
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Includes
|
||||
// ------------------------------------------------------
|
||||
|
||||
#include <stddef.h> // size_t
|
||||
#include <stdbool.h> // bool
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Standard malloc interface
|
||||
// ------------------------------------------------------
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);
|
||||
mi_decl_export void* mi_expand(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);
|
||||
|
||||
mi_decl_export void mi_free(void* p) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept mi_attr_malloc;
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept mi_attr_malloc;
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept;
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Extended allocation functions
|
||||
// ------------------------------------------------------
|
||||
#define MI_SMALL_WSIZE_MAX (128)
|
||||
#define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*))
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept;
|
||||
|
||||
// `mi_free_small` is for special applications like language runtimes.
|
||||
// it should only be used to free objects from `mi_(heap_)(m|z)alloc_small` and is potentially a tiny bit faster than `mi_free`
|
||||
mi_decl_export void mi_free_small(void* p) mi_attr_noexcept;
|
||||
|
||||
// -------------------------------------------------------------------------------------
|
||||
// Aligned allocation
|
||||
// Note that `alignment` always follows `size` for consistency with unaligned
|
||||
// allocation, but unfortunately this differs from `posix_memalign` and `aligned_alloc`.
|
||||
// -------------------------------------------------------------------------------------
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1, 2) mi_attr_alloc_align(3);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1, 2);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2);
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Typed allocation, the type is always the first parameter
|
||||
// ------------------------------------------------------
|
||||
|
||||
#define mi_malloc_tp(tp) ((tp*)mi_malloc(sizeof(tp)))
|
||||
#define mi_zalloc_tp(tp) ((tp*)mi_zalloc(sizeof(tp)))
|
||||
#define mi_calloc_tp(tp,n) ((tp*)mi_calloc(n,sizeof(tp)))
|
||||
#define mi_mallocn_tp(tp,n) ((tp*)mi_mallocn(n,sizeof(tp)))
|
||||
#define mi_reallocn_tp(tp,p,n) ((tp*)mi_reallocn(p,n,sizeof(tp)))
|
||||
#define mi_recalloc_tp(tp,p,n) ((tp*)mi_recalloc(p,n,sizeof(tp)))
|
||||
|
||||
#define mi_heap_malloc_tp(tp,hp) ((tp*)mi_heap_malloc(hp,sizeof(tp)))
|
||||
#define mi_heap_zalloc_tp(tp,hp) ((tp*)mi_heap_zalloc(hp,sizeof(tp)))
|
||||
#define mi_heap_calloc_tp(tp,hp,n) ((tp*)mi_heap_calloc(hp,n,sizeof(tp)))
|
||||
#define mi_heap_mallocn_tp(tp,hp,n) ((tp*)mi_heap_mallocn(hp,n,sizeof(tp)))
|
||||
#define mi_heap_reallocn_tp(tp,hp,p,n) ((tp*)mi_heap_reallocn(hp,p,n,sizeof(tp)))
|
||||
#define mi_heap_recalloc_tp(tp,hp,p,n) ((tp*)mi_heap_recalloc(hp,p,n,sizeof(tp)))
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Internals
|
||||
// See also `mimalloc-stats.h` for statistics
|
||||
// ------------------------------------------------------
|
||||
|
||||
typedef void (mi_cdecl mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg);
|
||||
mi_decl_export void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg) mi_attr_noexcept;
|
||||
|
||||
typedef void (mi_cdecl mi_output_fun)(const char* msg, void* arg);
|
||||
mi_decl_export void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept;
|
||||
|
||||
typedef void (mi_cdecl mi_error_fun)(int err, void* arg);
|
||||
mi_decl_export void mi_register_error(mi_error_fun* fun, void* arg);
|
||||
|
||||
mi_decl_export void mi_collect(bool force) mi_attr_noexcept;
|
||||
mi_decl_export int mi_version(void) mi_attr_noexcept;
|
||||
mi_decl_export void mi_options_print(void) mi_attr_noexcept;
|
||||
mi_decl_export void mi_process_info_print(void) mi_attr_noexcept;
|
||||
mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
|
||||
mi_decl_export void mi_process_info_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
|
||||
mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs,
|
||||
size_t* current_rss, size_t* peak_rss,
|
||||
size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept;
|
||||
|
||||
|
||||
|
||||
// Generally do not use the following as these are usually called automatically
|
||||
mi_decl_export void mi_process_init(void) mi_attr_noexcept;
|
||||
mi_decl_export void mi_cdecl mi_process_done(void) mi_attr_noexcept;
|
||||
mi_decl_export void mi_thread_init(void) mi_attr_noexcept;
|
||||
mi_decl_export void mi_thread_done(void) mi_attr_noexcept;
|
||||
mi_decl_export void mi_thread_set_in_threadpool(void) mi_attr_noexcept; // communicate that a thread is in a threadpool
|
||||
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
// Return allocated block size (if the return value is not NULL)
|
||||
// -----------------------------------------------------------------
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc(size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_urealloc(void* p, size_t newsize, size_t* block_size_pre, size_t* block_size_post) mi_attr_noexcept mi_attr_alloc_size(2);
|
||||
mi_decl_export void mi_ufree(void* p, size_t* block_size) mi_attr_noexcept;
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_uzalloc_small(size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
|
||||
|
||||
// -------------------------------------------------------------------------------------
|
||||
// Heaps: first-class. Can allocate from any thread (and be free'd from any thread)
|
||||
// Heaps keep allocations in separate pages from each other (but share the arena's and free'd pages)
|
||||
// -------------------------------------------------------------------------------------
|
||||
|
||||
struct mi_heap_s;
|
||||
typedef struct mi_heap_s mi_heap_t;
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new(void);
|
||||
mi_decl_export void mi_heap_delete(mi_heap_t* heap); // move live blocks to the main heap
|
||||
mi_decl_export void mi_heap_destroy(mi_heap_t* heap); // free all live blocks
|
||||
mi_decl_export void mi_heap_set_numa_affinity(mi_heap_t* heap, int numa_node);
|
||||
mi_decl_export void mi_heap_collect(mi_heap_t* heap, bool force);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_main(void);
|
||||
mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_of(const void* p);
|
||||
mi_decl_nodiscard mi_decl_export bool mi_heap_contains(const mi_heap_t* heap, const void* p);
|
||||
mi_decl_nodiscard mi_decl_export bool mi_any_heap_contains(const void* p);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc(mi_heap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3, 4);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_heap_reallocf(mi_heap_t* theap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept mi_attr_malloc;
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept mi_attr_malloc;
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept;
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3);
|
||||
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// Zero initialized re-allocation.
|
||||
// Only valid on memory that was originally allocated with zero initialization too.
|
||||
// e.g. `mi_calloc`, `mi_zalloc`, `mi_zalloc_aligned` etc.
|
||||
// see <https://github.com/microsoft/mimalloc/issues/63#issuecomment-508272992>
|
||||
// --------------------------------------------------------------------------------
|
||||
|
||||
mi_decl_nodiscard mi_decl_export void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_recalloc(void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(2,3) mi_attr_alloc_align(4);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(2,3);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3, 4);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(3, 4) mi_attr_alloc_align(5);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(3, 4);
|
||||
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Visiting pages and individual blocks in a heap.
|
||||
// ------------------------------------------------------
|
||||
|
||||
// An area of heap space contains blocks of a single size.
|
||||
typedef struct mi_heap_area_s {
|
||||
void* blocks; // start of the area containing theap blocks
|
||||
size_t reserved; // bytes reserved for this area (virtual)
|
||||
size_t committed; // current available bytes for this area
|
||||
size_t used; // number of allocated blocks
|
||||
size_t block_size; // size in bytes of each block
|
||||
size_t full_block_size; // size in bytes of a full block including padding and metadata.
|
||||
void* reserved1; // internal
|
||||
} mi_heap_area_t;
|
||||
|
||||
typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg);
|
||||
|
||||
mi_decl_export bool mi_heap_visit_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
|
||||
mi_decl_export bool mi_heap_visit_abandoned_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Arena memory management
|
||||
// Arena's are larger memory area's provided by the OS or user
|
||||
// ------------------------------------------------------
|
||||
|
||||
mi_decl_nodiscard mi_decl_export bool mi_is_redirected(void) mi_attr_noexcept;
|
||||
|
||||
mi_decl_export int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept;
|
||||
mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept;
|
||||
|
||||
mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept;
|
||||
mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_pinned /* cannot decommit/reset? */, bool is_zero, int numa_node) mi_attr_noexcept;
|
||||
|
||||
mi_decl_export void mi_debug_show_arenas(void) mi_attr_noexcept;
|
||||
mi_decl_export void mi_arenas_print(void) mi_attr_noexcept;
|
||||
mi_decl_export size_t mi_arena_min_alignment(void);
|
||||
mi_decl_export size_t mi_arena_min_size(void);
|
||||
|
||||
typedef void* mi_arena_id_t;
|
||||
mi_decl_export void* mi_arena_area(mi_arena_id_t arena_id, size_t* size);
|
||||
mi_decl_export int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;
|
||||
mi_decl_export int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;
|
||||
mi_decl_export bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;
|
||||
mi_decl_export bool mi_arena_contains(mi_arena_id_t arena_id, const void* p);
|
||||
|
||||
// Create a heap that only allocates in the specified arena
|
||||
mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id);
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Subprocesses
|
||||
// Advanced: allow sub-processes whose memory arena's stay fully separated (and no reclamation between them).
|
||||
// Used for example for separate interpreters in one process.
|
||||
// ------------------------------------------------------
|
||||
|
||||
typedef struct { void* _mi_subproc_id; } mi_subproc_id_t; // abstract type
|
||||
mi_decl_export mi_subproc_id_t mi_subproc_main(void);
|
||||
mi_decl_export mi_subproc_id_t mi_subproc_current(void);
|
||||
mi_decl_export mi_subproc_id_t mi_subproc_new(void);
|
||||
mi_decl_export void mi_subproc_destroy(mi_subproc_id_t subproc);
|
||||
mi_decl_export void mi_subproc_add_current_thread(mi_subproc_id_t subproc); // this should be called right after a thread is created (and no allocation has taken place yet)
|
||||
|
||||
typedef bool (mi_cdecl mi_heap_visit_fun)(mi_heap_t* heap, void* arg);
|
||||
mi_decl_export bool mi_subproc_visit_heaps(mi_subproc_id_t subproc, mi_heap_visit_fun* visitor, void* arg);
|
||||
|
||||
|
||||
// -------------------------------------------------------------------------------------
|
||||
// A "theap" is a thread-local heap. This API is only provided for special circumstances like runtimes
|
||||
// that already have a thread-local context and can store the theap there for (slightly) faster allocations.
|
||||
// This also allows to set a default theap for the current thread so that `malloc` etc. allocate from
|
||||
// that theap (instead of the main (t)heap).
|
||||
// Theaps are first-class, but can only allocate from the same thread that created it.
|
||||
// Allocation through a `theap` may be a tiny bit faster than using plain malloc
|
||||
// (as we don't need to lookup the thread local variable).
|
||||
// -------------------------------------------------------------------------------------
|
||||
|
||||
struct mi_theap_s;
|
||||
typedef struct mi_theap_s mi_theap_t;
|
||||
|
||||
mi_decl_export mi_theap_t* mi_heap_theap(mi_heap_t* heap);
|
||||
mi_decl_export mi_theap_t* mi_theap_set_default(mi_theap_t* theap);
|
||||
mi_decl_export mi_theap_t* mi_theap_get_default(void);
|
||||
mi_decl_export void mi_theap_collect(mi_theap_t* theap, bool force) mi_attr_noexcept;
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Experimental
|
||||
// ------------------------------------------------------
|
||||
|
||||
// Experimental: objects followed by a guard page.
|
||||
// Setting the sample rate on a specific theap can be used to test parts of the program more
|
||||
// specifically (in combination with `mi_theap_set_default`).
|
||||
// A sample rate of 0 disables guarded objects, while 1 uses a guard page for every object.
|
||||
// A seed of 0 uses a random start point. Only objects within the size bound are eligable for guard pages.
|
||||
mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed);
|
||||
mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max);
|
||||
|
||||
// very experimental
|
||||
typedef bool (mi_cdecl mi_commit_fun_t)(bool commit, void* start, size_t size, bool* is_zero, void* user_arg);
|
||||
mi_decl_export bool mi_manage_memory(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive,
|
||||
mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) mi_attr_noexcept;
|
||||
|
||||
//mi_decl_export bool mi_arena_unload(mi_arena_id_t arena_id, void** base, size_t* accessed_size, size_t* size);
|
||||
//mi_decl_export bool mi_arena_reload(void* start, size_t size, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id);
|
||||
//mi_decl_export bool mi_theap_reload(mi_theap_t* theap, mi_arena_id_t arena);
|
||||
//mi_decl_export void mi_theap_unload(mi_theap_t* theap);
|
||||
|
||||
// unsafe: assumes the page belonging to `p` is only accessed by the calling thread.
|
||||
mi_decl_export bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc_threshold) mi_attr_noexcept;
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Deprecated
|
||||
// ------------------------------------------------------
|
||||
|
||||
mi_decl_export bool mi_check_owned(const void* p);
|
||||
|
||||
mi_decl_export void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept;
|
||||
mi_decl_export bool mi_theap_visit_blocks(const mi_theap_t* theap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
|
||||
|
||||
mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept;
|
||||
mi_decl_export void mi_collect_reduce(size_t target_thread_owned) mi_attr_noexcept;
|
||||
|
||||
mi_decl_export void mi_stats_reset(void) mi_attr_noexcept;
|
||||
mi_decl_export void mi_stats_merge(void) mi_attr_noexcept;
|
||||
mi_decl_export void mi_stats_print(void* out) mi_attr_noexcept; // backward compatibility: `out` is ignored and should be NULL
|
||||
|
||||
mi_decl_export void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept; // not deprecated but declared in `mimalloc-stats.h` now.
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Options
|
||||
// ------------------------------------------------------
|
||||
|
||||
typedef enum mi_option_e {
|
||||
// stable options
|
||||
mi_option_show_errors, // print error messages
|
||||
mi_option_show_stats, // print statistics on termination
|
||||
mi_option_verbose, // print verbose messages
|
||||
// advanced options
|
||||
mi_option_deprecated_eager_commit,
|
||||
mi_option_arena_eager_commit, // eager commit arenas? Use 2 to enable just on overcommit systems (=2)
|
||||
mi_option_purge_decommits, // should a memory purge decommit? (=1). Set to 0 to use memory reset on a purge (instead of decommit)
|
||||
mi_option_allow_large_os_pages, // allow use of large (2 or 4 MiB) OS pages, implies eager commit.
|
||||
mi_option_reserve_huge_os_pages, // reserve N huge OS pages (1GiB pages) at startup
|
||||
mi_option_reserve_huge_os_pages_at, // reserve huge OS pages at a specific NUMA node
|
||||
mi_option_reserve_os_memory, // reserve specified amount of OS memory in an arena at startup (internally, this value is in KiB; use `mi_option_get_size`)
|
||||
mi_option_deprecated_segment_cache,
|
||||
mi_option_deprecated_page_reset,
|
||||
mi_option_deprecated_abandoned_page_purge,
|
||||
mi_option_deprecated_segment_reset,
|
||||
mi_option_deprecated_eager_commit_delay,
|
||||
mi_option_purge_delay, // memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all. (=10)
|
||||
mi_option_use_numa_nodes, // 0 = use all available numa nodes, otherwise use at most N nodes.
|
||||
mi_option_disallow_os_alloc, // 1 = do not use OS memory for allocation (but only programmatically reserved arenas)
|
||||
mi_option_os_tag, // tag used for OS logging (macOS only for now) (=100)
|
||||
mi_option_max_errors, // issue at most N error messages
|
||||
mi_option_max_warnings, // issue at most N warning messages
|
||||
mi_option_deprecated_max_segment_reclaim, // max. percentage of the abandoned segments can be reclaimed per try (=10%)
|
||||
mi_option_destroy_on_exit, // if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe
|
||||
mi_option_arena_reserve, // initial memory size for arena reservation (= 1 GiB on 64-bit) (internally, this value is in KiB; use `mi_option_get_size`)
|
||||
mi_option_arena_purge_mult, // multiplier for `purge_delay` for the purging delay for arenas (=10)
|
||||
mi_option_deprecated_purge_extend_delay,
|
||||
mi_option_disallow_arena_alloc, // 1 = do not use arena's for allocation (except if using specific arena id's)
|
||||
mi_option_retry_on_oom, // retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows)
|
||||
mi_option_visit_abandoned, // allow visiting theap blocks from abandoned threads (=0)
|
||||
mi_option_guarded_min, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects (=0)
|
||||
mi_option_guarded_max, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects (=0)
|
||||
mi_option_guarded_precise, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0)
|
||||
mi_option_guarded_sample_rate, // 1 out of N allocations in the min/max range will be guarded (=1000)
|
||||
mi_option_guarded_sample_seed, // can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0)
|
||||
mi_option_generic_collect, // collect theaps every N (=10000) generic allocation calls
|
||||
mi_option_page_reclaim_on_free, // reclaim abandoned pages on a free (=0). -1 disallowr always, 0 allows if the page originated from the current theap, 1 allow always
|
||||
mi_option_page_full_retain, // retain N full (small) pages per size class (=2)
|
||||
mi_option_page_max_candidates, // max candidate pages to consider for allocation (=4)
|
||||
mi_option_max_vabits, // max user space virtual address bits to consider (=48)
|
||||
mi_option_pagemap_commit, // commit the full pagemap (to always catch invalid pointer uses) (=0)
|
||||
mi_option_page_commit_on_demand, // commit page memory on-demand
|
||||
mi_option_page_max_reclaim, // don't reclaim pages of the same originating theap if we already own N pages (in that size class) (=-1 (unlimited))
|
||||
mi_option_page_cross_thread_max_reclaim, // don't reclaim pages across threads if we already own N pages (in that size class) (=16)
|
||||
mi_option_allow_thp, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process.
|
||||
mi_option_minimal_purge_size, // set minimal purge size (in KiB) (=0). By default set to either 64 or 2048 if THP is enabled.
|
||||
mi_option_arena_max_object_size, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit).
|
||||
mi_option_arena_is_numa_local, // experimental
|
||||
_mi_option_last,
|
||||
// legacy option names
|
||||
mi_option_large_os_pages = mi_option_allow_large_os_pages,
|
||||
mi_option_eager_region_commit = mi_option_arena_eager_commit,
|
||||
mi_option_reset_decommits = mi_option_purge_decommits,
|
||||
mi_option_reset_delay = mi_option_purge_delay,
|
||||
mi_option_limit_os_alloc = mi_option_disallow_os_alloc
|
||||
} mi_option_t;
|
||||
|
||||
|
||||
mi_decl_nodiscard mi_decl_export bool mi_option_is_enabled(mi_option_t option);
|
||||
mi_decl_export void mi_option_enable(mi_option_t option);
|
||||
mi_decl_export void mi_option_disable(mi_option_t option);
|
||||
mi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable);
|
||||
mi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option);
|
||||
mi_decl_nodiscard mi_decl_export long mi_option_get_clamp(mi_option_t option, long min, long max);
|
||||
mi_decl_nodiscard mi_decl_export size_t mi_option_get_size(mi_option_t option);
|
||||
mi_decl_export void mi_option_set(mi_option_t option, long value);
|
||||
mi_decl_export void mi_option_set_default(mi_option_t option, long value);
|
||||
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------
|
||||
// "mi" prefixed implementations of various posix, Unix, Windows, and C++ allocation functions.
|
||||
// (This can be convenient when providing overrides of these functions as done in `mimalloc-override.h`.)
|
||||
// note: we use `mi_cfree` as "checked free" and it checks if the pointer is in our theap before free-ing.
|
||||
// -------------------------------------------------------------------------------------------------------
|
||||
|
||||
mi_decl_export void mi_cfree(void* p) mi_attr_noexcept;
|
||||
mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept;
|
||||
|
||||
mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3);
|
||||
mi_decl_nodiscard mi_decl_export int mi_reallocarr(void* ptrp, size_t count, size_t size) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept;
|
||||
|
||||
mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept;
|
||||
mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept;
|
||||
mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept;
|
||||
mi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept;
|
||||
|
||||
// wide characters
|
||||
#include <wchar.h> // wchar_t
|
||||
mi_decl_export int mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s) mi_attr_noexcept mi_attr_malloc;
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept mi_attr_malloc;
|
||||
|
||||
// The `mi_new` wrappers implement C++ semantics on out-of-memory instead of directly returning `NULL`.
|
||||
// (and call `std::get_new_handler` and potentially raise a `std::bad_alloc` exception).
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new(size_t size) mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_n(size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(1, 2);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_new_realloc(void* p, size_t newsize) mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Implement the C++ std::allocator interface for use in STL containers.
|
||||
// (note: see `mimalloc-new-delete.h` for overriding the new/delete operators globally)
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
#ifdef __cplusplus
|
||||
|
||||
#include <cstddef> // std::size_t
|
||||
#include <cstdint> // PTRDIFF_MAX
|
||||
#if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11
|
||||
#include <type_traits> // std::true_type
|
||||
#include <utility> // std::forward
|
||||
#endif
|
||||
|
||||
template<class T> struct _mi_stl_allocator_common {
|
||||
typedef T value_type;
|
||||
typedef std::size_t size_type;
|
||||
typedef std::ptrdiff_t difference_type;
|
||||
typedef value_type& reference;
|
||||
typedef value_type const& const_reference;
|
||||
typedef value_type* pointer;
|
||||
typedef value_type const* const_pointer;
|
||||
|
||||
#if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11
|
||||
using propagate_on_container_copy_assignment = std::true_type;
|
||||
using propagate_on_container_move_assignment = std::true_type;
|
||||
using propagate_on_container_swap = std::true_type;
|
||||
template <class U, class ...Args> void construct(U* p, Args&& ...args) { ::new(p) U(std::forward<Args>(args)...); }
|
||||
template <class U> void destroy(U* p) mi_attr_noexcept { p->~U(); }
|
||||
#else
|
||||
void construct(pointer p, value_type const& val) { ::new(p) value_type(val); }
|
||||
void destroy(pointer p) { p->~value_type(); }
|
||||
#endif
|
||||
|
||||
size_type max_size() const mi_attr_noexcept { return (PTRDIFF_MAX/sizeof(value_type)); }
|
||||
pointer address(reference x) const { return &x; }
|
||||
const_pointer address(const_reference x) const { return &x; }
|
||||
};
|
||||
|
||||
template<class T> struct mi_stl_allocator : public _mi_stl_allocator_common<T> {
|
||||
using typename _mi_stl_allocator_common<T>::size_type;
|
||||
using typename _mi_stl_allocator_common<T>::value_type;
|
||||
using typename _mi_stl_allocator_common<T>::pointer;
|
||||
template <class U> struct rebind { typedef mi_stl_allocator<U> other; };
|
||||
|
||||
mi_stl_allocator() mi_attr_noexcept = default;
|
||||
mi_stl_allocator(const mi_stl_allocator&) mi_attr_noexcept = default;
|
||||
template<class U> mi_stl_allocator(const mi_stl_allocator<U>&) mi_attr_noexcept { }
|
||||
mi_stl_allocator select_on_container_copy_construction() const { return *this; }
|
||||
void deallocate(T* p, size_type) { mi_free(p); }
|
||||
|
||||
#if (__cplusplus >= 201703L) // C++17
|
||||
mi_decl_nodiscard T* allocate(size_type count) { return static_cast<T*>(mi_new_n(count, sizeof(T))); }
|
||||
mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); }
|
||||
#else
|
||||
mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast<pointer>(mi_new_n(count, sizeof(value_type))); }
|
||||
#endif
|
||||
|
||||
#if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11
|
||||
using is_always_equal = std::true_type;
|
||||
#endif
|
||||
};
|
||||
|
||||
template<class T1,class T2> bool operator==(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return true; }
|
||||
template<class T1,class T2> bool operator!=(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return false; }
|
||||
|
||||
|
||||
#if (__cplusplus >= 201103L) || (_MSC_VER >= 1900) // C++11
|
||||
#define MI_HAS_HEAP_STL_ALLOCATOR 1
|
||||
|
||||
#include <memory> // std::shared_ptr
|
||||
|
||||
// Common base class for STL allocators in a specific theap
|
||||
template<class T, bool _mi_destroy> struct _mi_heap_stl_allocator_common : public _mi_stl_allocator_common<T> {
|
||||
using typename _mi_stl_allocator_common<T>::size_type;
|
||||
using typename _mi_stl_allocator_common<T>::value_type;
|
||||
using typename _mi_stl_allocator_common<T>::pointer;
|
||||
|
||||
_mi_heap_stl_allocator_common(mi_heap_t* hp) : heap(hp, [](mi_heap_t*) {}) {} /* will not delete nor destroy the passed in heap */
|
||||
|
||||
#if (__cplusplus >= 201703L) // C++17
|
||||
mi_decl_nodiscard T* allocate(size_type count) { return static_cast<T*>(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(T))); }
|
||||
mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); }
|
||||
#else
|
||||
mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast<pointer>(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(value_type))); }
|
||||
#endif
|
||||
|
||||
#if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11
|
||||
using is_always_equal = std::false_type;
|
||||
#endif
|
||||
|
||||
void collect(bool force) { mi_heap_collect(this->heap.get(), force); }
|
||||
template<class U> bool is_equal(const _mi_heap_stl_allocator_common<U, _mi_destroy>& x) const { return (this->heap == x.heap); }
|
||||
|
||||
protected:
|
||||
std::shared_ptr<mi_heap_t> heap;
|
||||
template<class U, bool D> friend struct _mi_heap_stl_allocator_common;
|
||||
|
||||
_mi_heap_stl_allocator_common() {
|
||||
mi_heap_t* hp = mi_heap_new();
|
||||
this->heap.reset(hp, (_mi_destroy ? &heap_destroy : &heap_delete)); /* calls heap_delete/destroy when the refcount drops to zero */
|
||||
}
|
||||
_mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common& x) mi_attr_noexcept : heap(x.heap) { }
|
||||
template<class U> _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common<U, _mi_destroy>& x) mi_attr_noexcept : heap(x.heap) { }
|
||||
|
||||
private:
|
||||
static void heap_delete(mi_heap_t* hp) { if (hp != NULL) { mi_heap_delete(hp); } }
|
||||
static void heap_destroy(mi_heap_t* hp) { if (hp != NULL) { mi_heap_destroy(hp); } }
|
||||
};
|
||||
|
||||
// STL allocator allocation in a specific heap
|
||||
template<class T> struct mi_heap_stl_allocator : public _mi_heap_stl_allocator_common<T, false> {
|
||||
using typename _mi_heap_stl_allocator_common<T, false>::size_type;
|
||||
mi_heap_stl_allocator() : _mi_heap_stl_allocator_common<T, false>() { } // creates fresh heap that is deleted when the destructor is called
|
||||
mi_heap_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common<T, false>(hp) { } // no delete nor destroy on the passed in heap
|
||||
template<class U> mi_heap_stl_allocator(const mi_heap_stl_allocator<U>& x) mi_attr_noexcept : _mi_heap_stl_allocator_common<T, false>(x) { }
|
||||
|
||||
mi_heap_stl_allocator select_on_container_copy_construction() const { return *this; }
|
||||
void deallocate(T* p, size_type) { mi_free(p); }
|
||||
template<class U> struct rebind { typedef mi_heap_stl_allocator<U> other; };
|
||||
};
|
||||
|
||||
template<class T1, class T2> bool operator==(const mi_heap_stl_allocator<T1>& x, const mi_heap_stl_allocator<T2>& y) mi_attr_noexcept { return (x.is_equal(y)); }
|
||||
template<class T1, class T2> bool operator!=(const mi_heap_stl_allocator<T1>& x, const mi_heap_stl_allocator<T2>& y) mi_attr_noexcept { return (!x.is_equal(y)); }
|
||||
|
||||
|
||||
// STL allocator allocation in a specific heap, where `free` does nothing and
|
||||
// the heap is destroyed in one go on destruction -- use with care!
|
||||
template<class T> struct mi_heap_destroy_stl_allocator : public _mi_heap_stl_allocator_common<T, true> {
|
||||
using typename _mi_heap_stl_allocator_common<T, true>::size_type;
|
||||
mi_heap_destroy_stl_allocator() : _mi_heap_stl_allocator_common<T, true>() { } // creates fresh heap that is destroyed when the destructor is called
|
||||
mi_heap_destroy_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common<T, true>(hp) { } // no delete nor destroy on the passed in heap
|
||||
template<class U> mi_heap_destroy_stl_allocator(const mi_heap_destroy_stl_allocator<U>& x) mi_attr_noexcept : _mi_heap_stl_allocator_common<T, true>(x) { }
|
||||
|
||||
mi_heap_destroy_stl_allocator select_on_container_copy_construction() const { return *this; }
|
||||
void deallocate(T*, size_type) { /* do nothing as we destroy the heap on destruct. */ }
|
||||
template<class U> struct rebind { typedef mi_heap_destroy_stl_allocator<U> other; };
|
||||
};
|
||||
|
||||
template<class T1, class T2> bool operator==(const mi_heap_destroy_stl_allocator<T1>& x, const mi_heap_destroy_stl_allocator<T2>& y) mi_attr_noexcept { return (x.is_equal(y)); }
|
||||
template<class T1, class T2> bool operator!=(const mi_heap_destroy_stl_allocator<T1>& x, const mi_heap_destroy_stl_allocator<T2>& y) mi_attr_noexcept { return (!x.is_equal(y)); }
|
||||
|
||||
#endif // C++11
|
||||
|
||||
#endif // __cplusplus
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,562 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2024 Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#pragma once
|
||||
#ifndef MI_ATOMIC_H
|
||||
#define MI_ATOMIC_H
|
||||
|
||||
// include windows.h or pthreads.h
|
||||
#if defined(_WIN32)
|
||||
#ifndef WIN32_LEAN_AND_MEAN
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#endif
|
||||
#include <windows.h>
|
||||
#elif !defined(__wasi__) && (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__))
|
||||
#define MI_USE_PTHREADS
|
||||
#include <pthread.h>
|
||||
#endif
|
||||
|
||||
// --------------------------------------------------------------------------------------------
|
||||
// Atomics
|
||||
// We need to be portable between C, C++, and MSVC.
|
||||
// We base the primitives on the C/C++ atomics and create a minimal wrapper for MSVC in C compilation mode.
|
||||
// This is why we try to use only `uintptr_t` and `<type>*` as atomic types.
|
||||
// To gain better insight in the range of used atomics, we use explicitly named memory order operations
|
||||
// instead of passing the memory order as a parameter.
|
||||
// -----------------------------------------------------------------------------------------------
|
||||
|
||||
#if defined(__cplusplus)
|
||||
// Use C++ atomics
|
||||
#include <atomic>
|
||||
#define _Atomic(tp) std::atomic<tp>
|
||||
#define mi_atomic(name) std::atomic_##name
|
||||
#define mi_memory_order(name) std::memory_order_##name
|
||||
#if (__cplusplus >= 202002L) // c++20, see issue #571
|
||||
#define MI_ATOMIC_VAR_INIT(x) x
|
||||
#elif !defined(ATOMIC_VAR_INIT)
|
||||
#define MI_ATOMIC_VAR_INIT(x) x
|
||||
#else
|
||||
#define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x)
|
||||
#endif
|
||||
#elif defined(_MSC_VER)
|
||||
// Use MSVC C wrapper for C11 atomics
|
||||
#define _Atomic(tp) tp
|
||||
#define MI_ATOMIC_VAR_INIT(x) x
|
||||
#define mi_atomic(name) mi_atomic_##name
|
||||
#define mi_memory_order(name) mi_memory_order_##name
|
||||
#else
|
||||
// Use C11 atomics
|
||||
#include <stdatomic.h>
|
||||
#define mi_atomic(name) atomic_##name
|
||||
#define mi_memory_order(name) memory_order_##name
|
||||
#if (__STDC_VERSION__ >= 201710L) // c17, see issue #735
|
||||
#define MI_ATOMIC_VAR_INIT(x) x
|
||||
#elif !defined(ATOMIC_VAR_INIT)
|
||||
#define MI_ATOMIC_VAR_INIT(x) x
|
||||
#else
|
||||
#define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x)
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// Various defines for all used memory orders in mimalloc
|
||||
#define mi_atomic_cas_weak(p,expected,desired,mem_success,mem_fail) \
|
||||
mi_atomic(compare_exchange_weak_explicit)(p,expected,desired,mem_success,mem_fail)
|
||||
|
||||
#define mi_atomic_cas_strong(p,expected,desired,mem_success,mem_fail) \
|
||||
mi_atomic(compare_exchange_strong_explicit)(p,expected,desired,mem_success,mem_fail)
|
||||
|
||||
#define mi_atomic_load_acquire(p) mi_atomic(load_explicit)(p,mi_memory_order(acquire))
|
||||
#define mi_atomic_load_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed))
|
||||
#define mi_atomic_store_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release))
|
||||
#define mi_atomic_store_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed))
|
||||
#define mi_atomic_exchange_relaxed(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(relaxed))
|
||||
#define mi_atomic_exchange_release(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(release))
|
||||
#define mi_atomic_exchange_acq_rel(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(acq_rel))
|
||||
|
||||
#define mi_atomic_cas_weak_relaxed(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(relaxed),mi_memory_order(relaxed))
|
||||
#define mi_atomic_cas_weak_release(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed))
|
||||
#define mi_atomic_cas_weak_acq_rel(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire))
|
||||
#define mi_atomic_cas_strong_relaxed(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(relaxed),mi_memory_order(relaxed))
|
||||
#define mi_atomic_cas_strong_release(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed))
|
||||
#define mi_atomic_cas_strong_acq_rel(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire))
|
||||
|
||||
#define mi_atomic_add_relaxed(p,x) mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(relaxed))
|
||||
#define mi_atomic_add_acq_rel(p,x) mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(acq_rel))
|
||||
#define mi_atomic_sub_relaxed(p,x) mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(relaxed))
|
||||
#define mi_atomic_sub_acq_rel(p,x) mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(acq_rel))
|
||||
#define mi_atomic_and_relaxed(p,x) mi_atomic(fetch_and_explicit)(p,x,mi_memory_order(relaxed))
|
||||
#define mi_atomic_and_acq_rel(p,x) mi_atomic(fetch_and_explicit)(p,x,mi_memory_order(acq_rel))
|
||||
#define mi_atomic_or_relaxed(p,x) mi_atomic(fetch_or_explicit)(p,x,mi_memory_order(relaxed))
|
||||
#define mi_atomic_or_acq_rel(p,x) mi_atomic(fetch_or_explicit)(p,x,mi_memory_order(acq_rel))
|
||||
|
||||
#define mi_atomic_increment_relaxed(p) mi_atomic_add_relaxed(p,(uintptr_t)1)
|
||||
#define mi_atomic_decrement_relaxed(p) mi_atomic_sub_relaxed(p,(uintptr_t)1)
|
||||
#define mi_atomic_increment_acq_rel(p) mi_atomic_add_acq_rel(p,(uintptr_t)1)
|
||||
#define mi_atomic_decrement_acq_rel(p) mi_atomic_sub_acq_rel(p,(uintptr_t)1)
|
||||
|
||||
static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add);
|
||||
static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub);
|
||||
|
||||
|
||||
#if defined(__cplusplus) || !defined(_MSC_VER)
|
||||
|
||||
// In C++/C11 atomics we have polymorphic atomics so can use the typed `ptr` variants (where `tp` is the type of atomic value)
|
||||
// We use these macros so we can provide a typed wrapper in MSVC in C compilation mode as well
|
||||
#define mi_atomic_load_ptr_acquire(tp,p) mi_atomic_load_acquire(p)
|
||||
#define mi_atomic_load_ptr_relaxed(tp,p) mi_atomic_load_relaxed(p)
|
||||
|
||||
// In C++ we need to add casts to help resolve templates if NULL is passed
|
||||
#if defined(__cplusplus)
|
||||
#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release(p,(tp*)x)
|
||||
#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed(p,(tp*)x)
|
||||
#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,(tp*)des)
|
||||
#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,(tp*)des)
|
||||
#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,(tp*)des)
|
||||
#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des) mi_atomic_cas_strong_acq_rel(p,exp,(tp*)des)
|
||||
#define mi_atomic_exchange_ptr_relaxed(tp,p,x) mi_atomic_exchange_relaxed(p,(tp*)x)
|
||||
#define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,(tp*)x)
|
||||
#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,(tp*)x)
|
||||
#else
|
||||
#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release(p,x)
|
||||
#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed(p,x)
|
||||
#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,des)
|
||||
#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,des)
|
||||
#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,des)
|
||||
#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des) mi_atomic_cas_strong_acq_rel(p,exp,des)
|
||||
#define mi_atomic_exchange_ptr_relaxed(tp,p,x) mi_atomic_exchange_relaxed(p,x)
|
||||
#define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,x)
|
||||
#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,x)
|
||||
#endif
|
||||
|
||||
// These are used by the statistics
|
||||
static inline int64_t mi_atomic_addi64_relaxed(volatile int64_t* p, int64_t add) {
|
||||
return mi_atomic(fetch_add_explicit)((_Atomic(int64_t)*)p, add, mi_memory_order(relaxed));
|
||||
}
|
||||
static inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const volatile int64_t* padd) {
|
||||
const int64_t add = mi_atomic_load_relaxed((_Atomic(int64_t)*)padd);
|
||||
if (add != 0) {
|
||||
mi_atomic(fetch_add_explicit)((_Atomic(int64_t)*)p, add, mi_memory_order(relaxed));
|
||||
}
|
||||
}
|
||||
static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) {
|
||||
int64_t current = mi_atomic_load_relaxed((_Atomic(int64_t)*)p);
|
||||
while (current < x && !mi_atomic_cas_weak_release((_Atomic(int64_t)*)p, ¤t, x)) { /* nothing */ };
|
||||
}
|
||||
|
||||
// Used by timers
|
||||
#define mi_atomic_loadi64_acquire(p) mi_atomic(load_explicit)(p,mi_memory_order(acquire))
|
||||
#define mi_atomic_loadi64_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed))
|
||||
#define mi_atomic_storei64_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release))
|
||||
#define mi_atomic_storei64_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed))
|
||||
|
||||
#define mi_atomic_casi64_strong_acq_rel(p,e,d) mi_atomic_cas_strong_acq_rel(p,e,d)
|
||||
#define mi_atomic_addi64_acq_rel(p,i) mi_atomic_add_acq_rel(p,i)
|
||||
|
||||
|
||||
#elif defined(_MSC_VER)
|
||||
|
||||
// Deprecated: MSVC plain C compilation wrapper that uses Interlocked operations to model C11 atomics.
|
||||
// It is recommended to always compile as C++ when using MSVC.
|
||||
|
||||
#include <intrin.h>
|
||||
#ifdef _WIN64
|
||||
typedef LONG64 msc_intptr_t;
|
||||
#define MI_MSC_64(f) f##64
|
||||
#define MI_MSC_XX(f) f##64
|
||||
#else
|
||||
typedef LONG msc_intptr_t;
|
||||
#define MI_MSC_64(f) f
|
||||
#define MI_MSC_XX(f) f##32
|
||||
#endif
|
||||
|
||||
typedef enum mi_memory_order_e {
|
||||
mi_memory_order_relaxed,
|
||||
mi_memory_order_consume,
|
||||
mi_memory_order_acquire,
|
||||
mi_memory_order_release,
|
||||
mi_memory_order_acq_rel,
|
||||
mi_memory_order_seq_cst
|
||||
} mi_memory_order;
|
||||
|
||||
static inline uintptr_t mi_atomic_fetch_add_explicit(_Atomic(uintptr_t)*p, uintptr_t add, mi_memory_order mo) {
|
||||
(void)(mo);
|
||||
return (uintptr_t)MI_MSC_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, (msc_intptr_t)add);
|
||||
}
|
||||
static inline uintptr_t mi_atomic_fetch_sub_explicit(_Atomic(uintptr_t)*p, uintptr_t sub, mi_memory_order mo) {
|
||||
(void)(mo);
|
||||
return (uintptr_t)MI_MSC_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, -((msc_intptr_t)sub));
|
||||
}
|
||||
static inline uintptr_t mi_atomic_fetch_and_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {
|
||||
(void)(mo);
|
||||
return (uintptr_t)MI_MSC_64(_InterlockedAnd)((volatile msc_intptr_t*)p, (msc_intptr_t)x);
|
||||
}
|
||||
static inline uintptr_t mi_atomic_fetch_or_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {
|
||||
(void)(mo);
|
||||
return (uintptr_t)MI_MSC_64(_InterlockedOr)((volatile msc_intptr_t*)p, (msc_intptr_t)x);
|
||||
}
|
||||
static inline bool mi_atomic_compare_exchange_strong_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) {
|
||||
(void)(mo1); (void)(mo2);
|
||||
const uintptr_t read = (uintptr_t)MI_MSC_64(_InterlockedCompareExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)desired, (msc_intptr_t)(*expected));
|
||||
if (read == *expected) {
|
||||
return true;
|
||||
}
|
||||
else {
|
||||
*expected = read;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
static inline bool mi_atomic_compare_exchange_weak_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) {
|
||||
return mi_atomic_compare_exchange_strong_explicit(p, expected, desired, mo1, mo2);
|
||||
}
|
||||
static inline uintptr_t mi_atomic_exchange_explicit(_Atomic(uintptr_t)*p, uintptr_t exchange, mi_memory_order mo) {
|
||||
(void)(mo);
|
||||
return (uintptr_t)MI_MSC_64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange);
|
||||
}
|
||||
static inline void mi_atomic_thread_fence(mi_memory_order mo) {
|
||||
(void)(mo);
|
||||
_Atomic(uintptr_t) x = 0;
|
||||
mi_atomic_exchange_explicit(&x, 1, mo);
|
||||
}
|
||||
|
||||
static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_memory_order mo) {
|
||||
(void)(mo);
|
||||
// assert(mo<=mi_memory_order_acquire); // others are not used by mimalloc
|
||||
#if defined(_M_IX86) || defined(_M_X64)
|
||||
return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
|
||||
#elif defined(_M_ARM) || defined(_M_ARM64)
|
||||
if (mo == mi_memory_order_relaxed) {
|
||||
return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
|
||||
}
|
||||
else if (mo <= mi_memory_order_acquire) {
|
||||
return MI_MSC_XX(__ldar)((volatile const uintptr_t*)p);
|
||||
}
|
||||
else {
|
||||
const uintptr_t u = (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
|
||||
__dmb(15); // _ARM(64)_BARRIER_SY
|
||||
return u;
|
||||
}
|
||||
#else
|
||||
#warning "define mi_atomic_load_explicit for MSVC C compilation on this platform (which should be readonly, see issue #1277)"
|
||||
return MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
|
||||
#endif
|
||||
}
|
||||
static inline void mi_atomic_store_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {
|
||||
(void)(mo);
|
||||
// assert(mo<=mi_memory_order_release); // others are not used by mimalloc
|
||||
#if defined(_M_IX86) || defined(_M_X64)
|
||||
MI_MSC_XX(__iso_volatile_store)((volatile intptr_t*)p, x);
|
||||
#elif defined(_M_ARM) || defined(_M_ARM64)
|
||||
if (mo == mi_memory_order_relaxed) {
|
||||
MI_MSC_XX(__iso_volatile_store)((volatile intptr_t*)p, x);
|
||||
}
|
||||
else if (mo <= mi_memory_order_release) {
|
||||
MI_MSC_XX(__stlr)((volatile uintptr_t*)p,x);
|
||||
}
|
||||
else {
|
||||
mi_atomic_exchange_explicit(p, x, mo);
|
||||
}
|
||||
#else
|
||||
mi_atomic_exchange_explicit(p, x, mo);
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline int64_t mi_atomic_loadi64_explicit(_Atomic(int64_t)*p, mi_memory_order mo) {
|
||||
(void)(mo);
|
||||
// assert(mo<=mi_memory_order_acquire); // others are not used by mimalloc
|
||||
#if defined(_M_IX86) || defined(_M_X64)
|
||||
return __iso_volatile_load64((volatile const int64_t*)p);
|
||||
#elif defined(_M_ARM) || defined(_M_ARM64)
|
||||
if (mo == mi_memory_order_relaxed) {
|
||||
return __iso_volatile_load64((volatile const int64_t*)p);
|
||||
}
|
||||
#if defined(_M_ARM64)
|
||||
else if (mo <= mi_memory_order_acquire) {
|
||||
return __ldar64((volatile const uintptr_t*)p);
|
||||
}
|
||||
#endif
|
||||
else {
|
||||
const int64_t i = __iso_volatile_load64((volatile const int64_t*)p);
|
||||
__dmb(15); // _ARM(64)_BARRIER_SY
|
||||
return i;
|
||||
}
|
||||
#else
|
||||
#warning "define mi_atomic_loadi64_explicit for MSVC C compilation on this platform (which should be readonly, see issue #1277)"
|
||||
return __iso_volatile_load64((volatile const int64_t*)p);
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline void mi_atomic_storei64_explicit(_Atomic(int64_t)*p, int64_t x, mi_memory_order mo) {
|
||||
(void)(mo);
|
||||
// assert(mo<=mi_memory_order_release); // others are not used by mimalloc
|
||||
#if defined(_M_IX86) || defined(_M_X64)
|
||||
__iso_volatile_store64((volatile int64_t*)p,x);
|
||||
#elif defined(_M_ARM) || defined(_M_ARM64)
|
||||
if (mo == mi_memory_order_relaxed) {
|
||||
__iso_volatile_store64((volatile int64_t*)p,x);
|
||||
}
|
||||
#if defined(_M_ARM64)
|
||||
else if (mo == mi_memory_order_release) {
|
||||
__stlr64((volatile uint64_t*)p, (uint64_t)x);
|
||||
}
|
||||
#endif
|
||||
else {
|
||||
InterlockedExchange64(p, x);
|
||||
}
|
||||
#else
|
||||
InterlockedExchange64(p, x);
|
||||
#endif
|
||||
}
|
||||
|
||||
// These are used by the statistics
|
||||
static inline int64_t mi_atomic_addi64_relaxed(volatile _Atomic(int64_t)*p, int64_t add) {
|
||||
#ifdef _WIN64
|
||||
return (int64_t)mi_atomic_addi((int64_t*)p, add);
|
||||
#elif defined(_M_ARM)
|
||||
return _InterlockedExchangeAdd64(p, add);
|
||||
#else
|
||||
// x86
|
||||
int64_t current;
|
||||
int64_t sum;
|
||||
do {
|
||||
current = __iso_volatile_load64((volatile const int64_t*)p);
|
||||
sum = current + add;
|
||||
} while (_InterlockedCompareExchange64(p, sum, current) != current);
|
||||
return current;
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const volatile int64_t* padd) {
|
||||
const int64_t add = *padd;
|
||||
if (add != 0) {
|
||||
mi_atomic_addi64_relaxed((volatile _Atomic(int64_t)*)p, add);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)*p, int64_t x) {
|
||||
int64_t current;
|
||||
do {
|
||||
current = *p;
|
||||
} while (current < x && _InterlockedCompareExchange64(p, x, current) != current);
|
||||
}
|
||||
|
||||
static inline void mi_atomic_addi64_acq_rel(volatile _Atomic(int64_t*)p, int64_t i) {
|
||||
mi_atomic_addi64_relaxed(p, i);
|
||||
}
|
||||
|
||||
static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t*)p, int64_t* exp, int64_t des) {
|
||||
const int64_t read = _InterlockedCompareExchange64(p, des, *exp);
|
||||
if (read == *exp) {
|
||||
return true;
|
||||
}
|
||||
else {
|
||||
*exp = read;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// The pointer macros cast to `uintptr_t`.
|
||||
#define mi_atomic_load_ptr_acquire(tp,p) (tp*)mi_atomic_load_acquire((_Atomic(uintptr_t)*)(p))
|
||||
#define mi_atomic_load_ptr_relaxed(tp,p) (tp*)mi_atomic_load_relaxed((_Atomic(uintptr_t)*)(p))
|
||||
#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release((_Atomic(uintptr_t)*)(p),(uintptr_t)(x))
|
||||
#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)(x))
|
||||
#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
|
||||
#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
|
||||
#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
|
||||
#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des) mi_atomic_cas_strong_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
|
||||
#define mi_atomic_exchange_ptr_relaxed(tp,p,x) (tp*)mi_atomic_exchange_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)x)
|
||||
#define mi_atomic_exchange_ptr_release(tp,p,x) (tp*)mi_atomic_exchange_release((_Atomic(uintptr_t)*)(p),(uintptr_t)x)
|
||||
#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) (tp*)mi_atomic_exchange_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t)x)
|
||||
|
||||
#define mi_atomic_loadi64_acquire(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(acquire))
|
||||
#define mi_atomic_loadi64_relaxed(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(relaxed))
|
||||
#define mi_atomic_storei64_release(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(release))
|
||||
#define mi_atomic_storei64_relaxed(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(relaxed))
|
||||
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
// Atomically add a signed value; returns the previous value.
|
||||
static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add) {
|
||||
return (intptr_t)mi_atomic_add_acq_rel((_Atomic(uintptr_t)*)p, (uintptr_t)add);
|
||||
}
|
||||
|
||||
// Atomically subtract a signed value; returns the previous value.
|
||||
static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub) {
|
||||
return (intptr_t)mi_atomic_addi(p, -sub);
|
||||
}
|
||||
|
||||
|
||||
// ----------------------------------------------------------------------
|
||||
// Guard
|
||||
// ----------------------------------------------------------------------
|
||||
|
||||
typedef _Atomic(uintptr_t) mi_atomic_guard_t;
|
||||
|
||||
// Allows only one thread to execute at a time (without blocking anyone)
|
||||
#define mi_atomic_guard(guard) \
|
||||
uintptr_t _mi_guard_expected = 0; \
|
||||
for(bool _mi_guard_once = true; \
|
||||
_mi_guard_once && mi_atomic_cas_strong_acq_rel(guard,&_mi_guard_expected,(uintptr_t)1); \
|
||||
(mi_atomic_store_release(guard,(uintptr_t)0), _mi_guard_once = false) )
|
||||
|
||||
|
||||
// ----------------------------------------------------------------------
|
||||
// Locks
|
||||
// These should be light-weight in-process only locks.
|
||||
// Only used for reserving arena's and to maintain the abandoned list.
|
||||
// ----------------------------------------------------------------------
|
||||
#if _MSC_VER
|
||||
#pragma warning(disable:26110) // unlock with holding lock
|
||||
#endif
|
||||
|
||||
#define mi_lock(lock) for(bool _mi_go = (mi_lock_acquire(lock),true); _mi_go; (mi_lock_release(lock), _mi_go=false) )
|
||||
#define mi_lock_maybe(lock,acquire) for(bool _mi_go = (acquire ? (mi_lock_acquire(lock),true) : true); _mi_go; _mi_go = (acquire ? (mi_lock_release(lock),false) : false) )
|
||||
|
||||
|
||||
#if defined(_WIN32)
|
||||
|
||||
typedef struct mi_lock_s {
|
||||
SRWLOCK mutex; // slim reader-writer lock
|
||||
} mi_lock_t;
|
||||
|
||||
#define MI_LOCK_INITIALIZER { SRWLOCK_INIT }
|
||||
|
||||
static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
|
||||
return TryAcquireSRWLockExclusive(&lock->mutex);
|
||||
}
|
||||
static inline void mi_lock_acquire(mi_lock_t* lock) {
|
||||
AcquireSRWLockExclusive(&lock->mutex);
|
||||
}
|
||||
static inline void mi_lock_release(mi_lock_t* lock) {
|
||||
ReleaseSRWLockExclusive(&lock->mutex);
|
||||
}
|
||||
static inline void mi_lock_init(mi_lock_t* lock) {
|
||||
InitializeSRWLock(&lock->mutex);
|
||||
}
|
||||
static inline void mi_lock_done(mi_lock_t* lock) {
|
||||
(void)(lock);
|
||||
}
|
||||
|
||||
#elif defined(MI_USE_PTHREADS)
|
||||
|
||||
#include <string.h> // memcpy
|
||||
void _mi_error_message(int err, const char* fmt, ...);
|
||||
|
||||
typedef struct mi_lock_s {
|
||||
pthread_mutex_t mutex;
|
||||
} mi_lock_t;
|
||||
|
||||
#define MI_LOCK_INITIALIZER { PTHREAD_MUTEX_INITIALIZER }
|
||||
|
||||
static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
|
||||
return (pthread_mutex_trylock(&lock->mutex) == 0);
|
||||
}
|
||||
static inline void mi_lock_acquire(mi_lock_t* lock) {
|
||||
const int err = pthread_mutex_lock(&lock->mutex);
|
||||
if (err != 0) {
|
||||
_mi_error_message(err, "internal error: lock cannot be acquired (err %i)\n", err);
|
||||
}
|
||||
}
|
||||
static inline void mi_lock_release(mi_lock_t* lock) {
|
||||
pthread_mutex_unlock(&lock->mutex);
|
||||
}
|
||||
static inline void mi_lock_init(mi_lock_t* lock) {
|
||||
if(lock==NULL) return;
|
||||
// use this instead of pthread_mutex_init since that can cause allocation on some platforms (and recursively initialize)
|
||||
const pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
memcpy(&lock->mutex,&mutex,sizeof(mutex));
|
||||
}
|
||||
static inline void mi_lock_done(mi_lock_t* lock) {
|
||||
pthread_mutex_destroy(&lock->mutex);
|
||||
}
|
||||
|
||||
#elif defined(__cplusplus)
|
||||
|
||||
#include <thread>
|
||||
#include <mutex>
|
||||
#include <new>
|
||||
|
||||
typedef struct mi_lock_s {
|
||||
std::mutex mutex;
|
||||
} mi_lock_t;
|
||||
|
||||
#define MI_LOCK_INITIALIZER { }
|
||||
|
||||
static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
|
||||
return lock->mutex.try_lock();
|
||||
}
|
||||
static inline void mi_lock_acquire(mi_lock_t* lock) {
|
||||
lock->mutex.lock();
|
||||
}
|
||||
static inline void mi_lock_release(mi_lock_t* lock) {
|
||||
lock->mutex.unlock();
|
||||
}
|
||||
static inline void mi_lock_init(mi_lock_t* lock) {
|
||||
new(&lock->mutex) std::mutex();
|
||||
}
|
||||
static inline void mi_lock_done(mi_lock_t* lock) {
|
||||
(void)(lock);
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
// fall back to poor man's locks.
|
||||
// this should only be the case in a single-threaded environment (like __wasi__)
|
||||
#include <errno.h>
|
||||
#ifndef EFAULT
|
||||
#define EFAULT (14)
|
||||
#endif
|
||||
void _mi_error_message(int err, const char* fmt, ...);
|
||||
void _mi_prim_thread_yield(void);
|
||||
|
||||
typedef struct mi_lock_s {
|
||||
_Atomic(uintptr_t) mutex;
|
||||
} mi_lock_t;
|
||||
|
||||
#define MI_LOCK_INITIALIZER { MI_ATOMIC_VAR_INIT(0) }
|
||||
|
||||
static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
|
||||
uintptr_t expected = 0;
|
||||
return mi_atomic_cas_strong_acq_rel(&lock->mutex, &expected, (uintptr_t)1);
|
||||
}
|
||||
static inline void mi_lock_acquire(mi_lock_t* lock) {
|
||||
for (int i = 0; i < 10000; i++) { // for at most 10000 tries?
|
||||
if (mi_lock_try_acquire(lock)) return;
|
||||
_mi_prim_thread_yield();
|
||||
}
|
||||
_mi_error_message(EFAULT, "internal error: lock cannot be acquired (due to lack of native lock primitives)\n");
|
||||
}
|
||||
static inline void mi_lock_release(mi_lock_t* lock) {
|
||||
mi_atomic_store_release(&lock->mutex, (uintptr_t)0);
|
||||
}
|
||||
static inline void mi_lock_init(mi_lock_t* lock) {
|
||||
mi_lock_release(lock);
|
||||
}
|
||||
static inline void mi_lock_done(mi_lock_t* lock) {
|
||||
(void)(lock);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
typedef struct mi_atomic_once_s {
|
||||
_Atomic(uintptr_t) tid;
|
||||
mi_lock_t lock;
|
||||
} mi_atomic_once_t;
|
||||
|
||||
// Returns `true` only on the first invocation, signifying we can execute an action once.
|
||||
// If it returns `true`, the caller should call `_mi_atomic_once_release` after performing the action.
|
||||
// Other threads (than the initial thread that entered) will block until `_mi_atomic_once_release` has been called.
|
||||
bool _mi_atomic_once_enter(mi_atomic_once_t* once); // defined in `libc.c`
|
||||
void _mi_atomic_once_release(mi_atomic_once_t* once); // defined in `libc.c`
|
||||
|
||||
#define mi_atomic_do_once \
|
||||
static mi_atomic_once_t _mi_once = { MI_ATOMIC_VAR_INIT(0), MI_LOCK_INITIALIZER }; \
|
||||
for(bool _mi_exec = _mi_atomic_once_enter(&_mi_once); _mi_exec; (_mi_atomic_once_release(&_mi_once),_mi_exec=false))
|
||||
|
||||
|
||||
#endif // __MIMALLOC_ATOMIC_H
|
||||
@@ -0,0 +1,340 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2019-2024 Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
Bit operation, and platform dependent definition (MI_INTPTR_SIZE etc)
|
||||
---------------------------------------------------------------------------- */
|
||||
|
||||
#pragma once
|
||||
#ifndef MI_BITS_H
|
||||
#define MI_BITS_H
|
||||
|
||||
#include <stddef.h> // size_t
|
||||
#include <stdint.h> // int64_t etc
|
||||
#include <stdbool.h> // bool
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Size of a pointer.
|
||||
// We assume that `sizeof(void*)==sizeof(intptr_t)`
|
||||
// and it holds for all platforms we know of.
|
||||
//
|
||||
// However, the C standard only requires that:
|
||||
// p == (void*)((intptr_t)p))
|
||||
// but we also need:
|
||||
// i == (intptr_t)((void*)i)
|
||||
// or otherwise one might define an intptr_t type that is larger than a pointer...
|
||||
// ------------------------------------------------------
|
||||
|
||||
#if INTPTR_MAX > INT64_MAX
|
||||
# define MI_INTPTR_SHIFT (4) // assume 128-bit (as on arm CHERI for example)
|
||||
#elif INTPTR_MAX == INT64_MAX
|
||||
# define MI_INTPTR_SHIFT (3)
|
||||
#elif INTPTR_MAX == INT32_MAX
|
||||
# define MI_INTPTR_SHIFT (2)
|
||||
#else
|
||||
#error platform pointers must be 32, 64, or 128 bits
|
||||
#endif
|
||||
|
||||
#if (INTPTR_MAX) > LONG_MAX
|
||||
# define MI_PU(x) x##ULL
|
||||
#else
|
||||
# define MI_PU(x) x##UL
|
||||
#endif
|
||||
|
||||
#if SIZE_MAX == UINT64_MAX
|
||||
# define MI_SIZE_SHIFT (3)
|
||||
typedef int64_t mi_ssize_t;
|
||||
#elif SIZE_MAX == UINT32_MAX
|
||||
# define MI_SIZE_SHIFT (2)
|
||||
typedef int32_t mi_ssize_t;
|
||||
#else
|
||||
#error platform objects must be 32 or 64 bits in size
|
||||
#endif
|
||||
|
||||
#if (SIZE_MAX/2) > LONG_MAX
|
||||
# define MI_ZU(x) x##ULL
|
||||
#else
|
||||
# define MI_ZU(x) x##UL
|
||||
#endif
|
||||
|
||||
#define MI_INTPTR_SIZE (1<<MI_INTPTR_SHIFT)
|
||||
#define MI_INTPTR_BITS (MI_INTPTR_SIZE*8)
|
||||
|
||||
#define MI_SIZE_SIZE (1<<MI_SIZE_SHIFT)
|
||||
#define MI_SIZE_BITS (MI_SIZE_SIZE*8)
|
||||
|
||||
#define MI_KiB (MI_ZU(1024))
|
||||
#define MI_MiB (MI_KiB*MI_KiB)
|
||||
#define MI_GiB (MI_MiB*MI_KiB)
|
||||
|
||||
|
||||
/* --------------------------------------------------------------------------------
|
||||
Architecture
|
||||
-------------------------------------------------------------------------------- */
|
||||
|
||||
#if defined(__aarch64__) || defined(_M_ARM64) || defined(_M_HYBRID_X86_ARM64) || defined(_M_ARM64EC) // consider arm64ec as arm64
|
||||
#define MI_ARCH_ARM64 1
|
||||
#elif defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64)
|
||||
#define MI_ARCH_X64 1
|
||||
#elif defined(__i386__) || defined(__i386) || defined(_M_IX86) || defined(_X86_) || defined(__X86__)
|
||||
#define MI_ARCH_X86 1
|
||||
#elif defined(__arm__) || defined(_ARM) || defined(_M_ARM) || defined(_M_ARMT) || defined(__arm)
|
||||
#define MI_ARCH_ARM32 1
|
||||
#elif defined(__riscv) || defined(_M_RISCV)
|
||||
#define MI_ARCH_RISCV 1
|
||||
#if (LONG_MAX == INT32_MAX)
|
||||
#define MI_ARCH_RISCV32 1
|
||||
#else
|
||||
#define MI_ARCH_RISCV64 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if MI_ARCH_X64 && defined(__AVX2__)
|
||||
#include <immintrin.h>
|
||||
#elif MI_ARCH_ARM64 && MI_OPT_SIMD
|
||||
#include <arm_neon.h>
|
||||
#endif
|
||||
#if defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
|
||||
#include <intrin.h>
|
||||
#endif
|
||||
|
||||
#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__BMI2__) // msvc
|
||||
#define __BMI2__ 1
|
||||
#endif
|
||||
#if MI_ARCH_X64 && (defined(__AVX2__) || defined(__BMI2__)) && !defined(__BMI1__) // msvc
|
||||
#define __BMI1__ 1
|
||||
#endif
|
||||
#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__LZCNT__) // msvc
|
||||
#define __LZCNT__ 1
|
||||
#endif
|
||||
|
||||
// Define big endian if needed
|
||||
// #define MI_BIG_ENDIAN 1
|
||||
|
||||
// maximum virtual address bits in a user-space pointer
|
||||
#if MI_DEFAULT_VIRTUAL_ADDRESS_BITS > 0
|
||||
#define MI_MAX_VABITS MI_DEFAULT_VIRTUAL_ADDRESS_BITS
|
||||
#elif MI_ARCH_X64
|
||||
#define MI_MAX_VABITS (47)
|
||||
#elif MI_INTPTR_SIZE > 4
|
||||
#define MI_MAX_VABITS (48)
|
||||
#else
|
||||
#define MI_MAX_VABITS (32)
|
||||
#endif
|
||||
|
||||
// use a flat page-map or a 2-level one
|
||||
#ifndef MI_PAGE_MAP_FLAT
|
||||
#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && MI_SECURE==0 && !MI_PAGE_META_IS_SEPARATED
|
||||
#define MI_PAGE_MAP_FLAT 1
|
||||
#else
|
||||
#define MI_PAGE_MAP_FLAT 0
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
||||
/* --------------------------------------------------------------------------------
|
||||
Builtin's
|
||||
-------------------------------------------------------------------------------- */
|
||||
|
||||
#ifndef __has_builtin
|
||||
#define __has_builtin(x) 0
|
||||
#endif
|
||||
|
||||
#define mi_builtin(name) __builtin_##name
|
||||
#define mi_has_builtin(name) __has_builtin(__builtin_##name)
|
||||
|
||||
#if (LONG_MAX == INT32_MAX)
|
||||
#define mi_builtin32(name) mi_builtin(name##l)
|
||||
#define mi_has_builtin32(name) mi_has_builtin(name##l)
|
||||
#else
|
||||
#define mi_builtin32(name) mi_builtin(name)
|
||||
#define mi_has_builtin32(name) mi_has_builtin(name)
|
||||
#endif
|
||||
#if (LONG_MAX == INT64_MAX)
|
||||
#define mi_builtin64(name) mi_builtin(name##l)
|
||||
#define mi_has_builtin64(name) mi_has_builtin(name##l)
|
||||
#else
|
||||
#define mi_builtin64(name) mi_builtin(name##ll)
|
||||
#define mi_has_builtin64(name) mi_has_builtin(name##ll)
|
||||
#endif
|
||||
|
||||
#if (MI_SIZE_BITS == 32)
|
||||
#define mi_builtinz(name) mi_builtin32(name)
|
||||
#define mi_has_builtinz(name) mi_has_builtin32(name)
|
||||
#define mi_msc_builtinz(name) name
|
||||
#elif (MI_SIZE_BITS == 64)
|
||||
#define mi_builtinz(name) mi_builtin64(name)
|
||||
#define mi_has_builtinz(name) mi_has_builtin64(name)
|
||||
#define mi_msc_builtinz(name) name##64
|
||||
#endif
|
||||
|
||||
/* --------------------------------------------------------------------------------
|
||||
Popcount and count trailing/leading zero's
|
||||
-------------------------------------------------------------------------------- */
|
||||
|
||||
size_t _mi_popcount_generic(size_t x);
|
||||
|
||||
static inline size_t mi_popcount(size_t x) {
|
||||
#if mi_has_builtinz(popcount)
|
||||
return mi_builtinz(popcount)(x);
|
||||
#elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
|
||||
return mi_msc_builtinz(__popcnt)(x);
|
||||
#elif MI_ARCH_X64 && defined(__BMI1__)
|
||||
return (size_t)_mm_popcnt_u64(x);
|
||||
#else
|
||||
#define MI_HAS_FAST_POPCOUNT 0
|
||||
return (x<=1 ? x : _mi_popcount_generic(x));
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifndef MI_HAS_FAST_POPCOUNT
|
||||
#define MI_HAS_FAST_POPCOUNT 1
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
size_t _mi_clz_generic(size_t x);
|
||||
size_t _mi_ctz_generic(size_t x);
|
||||
|
||||
static inline size_t mi_ctz(size_t x) {
|
||||
#if defined(__GNUC__) && MI_ARCH_X64 && defined(__BMI1__) // on x64 tzcnt is defined for 0
|
||||
size_t r;
|
||||
__asm ("tzcnt\t%1, %0" : "=r"(r) : "r"(x) : "cc");
|
||||
return r;
|
||||
#elif defined(_MSC_VER) && MI_ARCH_X64 && defined(__BMI1__)
|
||||
return _tzcnt_u64(x);
|
||||
#elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
|
||||
unsigned long idx;
|
||||
return (mi_msc_builtinz(_BitScanForward)(&idx, x) ? (size_t)idx : MI_SIZE_BITS);
|
||||
#elif mi_has_builtinz(ctz)
|
||||
return (x!=0 ? (size_t)mi_builtinz(ctz)(x) : MI_SIZE_BITS);
|
||||
#elif defined(__GNUC__) && (MI_ARCH_X64 || MI_ARCH_X86)
|
||||
size_t r = MI_SIZE_BITS; // bsf leaves destination unmodified if the argument is 0 (see <https://github.com/llvm/llvm-project/pull/102885>)
|
||||
__asm ("bsf\t%1, %0" : "+r"(r) : "r"(x) : "cc");
|
||||
return r;
|
||||
#elif MI_HAS_FAST_POPCOUNT
|
||||
return (x!=0 ? (mi_popcount(x^(x-1))-1) : MI_SIZE_BITS);
|
||||
#else
|
||||
#define MI_HAS_FAST_BITSCAN 0
|
||||
return (x!=0 ? _mi_ctz_generic(x) : MI_SIZE_BITS);
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline size_t mi_clz(size_t x) {
|
||||
#if defined(__GNUC__) && MI_ARCH_X64 && defined(__LZCNT__) // on x64 lzcnt is defined for 0
|
||||
size_t r;
|
||||
__asm ("lzcnt\t%1, %0" : "=r"(r) : "r"(x) : "cc");
|
||||
return r;
|
||||
#elif defined(_MSC_VER) && MI_ARCH_X64 && defined(__LZCNT__)
|
||||
return _lzcnt_u64(x);
|
||||
#elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
|
||||
unsigned long idx;
|
||||
return (mi_msc_builtinz(_BitScanReverse)(&idx, x) ? MI_SIZE_BITS - 1 - (size_t)idx : MI_SIZE_BITS);
|
||||
#elif mi_has_builtinz(clz)
|
||||
return (x!=0 ? (size_t)mi_builtinz(clz)(x) : MI_SIZE_BITS);
|
||||
#elif defined(__GNUC__) && (MI_ARCH_X64 || MI_ARCH_X86)
|
||||
if (x==0) return MI_SIZE_BITS;
|
||||
size_t r;
|
||||
__asm ("bsr\t%1, %0" : "=r"(r) : "r"(x) : "cc");
|
||||
return (MI_SIZE_BITS - 1 - r);
|
||||
#else
|
||||
#define MI_HAS_FAST_BITSCAN 0
|
||||
return (x!=0 ? _mi_clz_generic(x) : MI_SIZE_BITS);
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifndef MI_HAS_FAST_BITSCAN
|
||||
#define MI_HAS_FAST_BITSCAN 1
|
||||
#endif
|
||||
|
||||
/* --------------------------------------------------------------------------------
|
||||
find trailing/leading zero (bit scan forward/reverse)
|
||||
-------------------------------------------------------------------------------- */
|
||||
|
||||
// Bit scan forward: find the least significant bit that is set (i.e. count trailing zero's)
|
||||
// return false if `x==0` (with `*idx` undefined) and true otherwise,
|
||||
// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`).
|
||||
static inline bool mi_bsf(size_t x, size_t* idx) {
|
||||
#if defined(__GNUC__) && MI_ARCH_X64 && defined(__BMI1__) && (!defined(__clang_major__) || __clang_major__ >= 9)
|
||||
// on x64 the carry flag is set on zero which gives better codegen
|
||||
bool is_zero;
|
||||
__asm ( "tzcnt\t%2, %1" : "=@ccc"(is_zero), "=r"(*idx) : "r"(x) : "cc" );
|
||||
return !is_zero;
|
||||
#elif 0 && defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
|
||||
unsigned long i;
|
||||
return (mi_msc_builtinz(_BitScanForward)(&i, x) ? (*idx = (size_t)i, true) : false);
|
||||
#else
|
||||
return (x!=0 ? (*idx = mi_ctz(x), true) : false);
|
||||
#endif
|
||||
}
|
||||
|
||||
// Bit scan reverse: find the most significant bit that is set
|
||||
// return false if `x==0` (with `*idx` undefined) and true otherwise,
|
||||
// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`).
|
||||
static inline bool mi_bsr(size_t x, size_t* idx) {
|
||||
#if 0 && defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
|
||||
unsigned long i;
|
||||
return (mi_msc_builtinz(_BitScanReverse)(&i, x) ? (*idx = (size_t)i, true) : false);
|
||||
#else
|
||||
return (x!=0 ? (*idx = MI_SIZE_BITS - 1 - mi_clz(x), true) : false);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
/* --------------------------------------------------------------------------------
|
||||
rotate
|
||||
-------------------------------------------------------------------------------- */
|
||||
|
||||
static inline size_t mi_rotr(size_t x, size_t r) {
|
||||
#if (mi_has_builtin(rotateright64) && MI_SIZE_BITS==64)
|
||||
return mi_builtin(rotateright64)(x,r);
|
||||
#elif (mi_has_builtin(rotateright32) && MI_SIZE_BITS==32)
|
||||
return mi_builtin(rotateright32)(x,r);
|
||||
#elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_ARM64)
|
||||
return _rotr64(x, (int)r);
|
||||
#elif defined(_MSC_VER) && (MI_ARCH_X86 || MI_ARCH_ARM32)
|
||||
return _lrotr(x,(int)r);
|
||||
#else
|
||||
// The term `(-rshift)&(BITS-1)` is written instead of `BITS - rshift` to
|
||||
// avoid UB when `rshift==0`. See <https://blog.regehr.org/archives/1063>
|
||||
const unsigned int rshift = (unsigned int)(r) & (MI_SIZE_BITS-1);
|
||||
return ((x >> rshift) | (x << ((-rshift) & (MI_SIZE_BITS-1))));
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline size_t mi_rotl(size_t x, size_t r) {
|
||||
#if (mi_has_builtin(rotateleft64) && MI_SIZE_BITS==64)
|
||||
return mi_builtin(rotateleft64)(x,r);
|
||||
#elif (mi_has_builtin(rotateleft32) && MI_SIZE_BITS==32)
|
||||
return mi_builtin(rotateleft32)(x,r);
|
||||
#elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_ARM64)
|
||||
return _rotl64(x, (int)r);
|
||||
#elif defined(_MSC_VER) && (MI_ARCH_X86 || MI_ARCH_ARM32)
|
||||
return _lrotl(x, (int)r);
|
||||
#else
|
||||
// The term `(-rshift)&(BITS-1)` is written instead of `BITS - rshift` to
|
||||
// avoid UB when `rshift==0`. See <https://blog.regehr.org/archives/1063>
|
||||
const unsigned int rshift = (unsigned int)(r) & (MI_SIZE_BITS-1);
|
||||
return ((x << rshift) | (x >> ((-rshift) & (MI_SIZE_BITS-1))));
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline uint32_t mi_rotl32(uint32_t x, uint32_t r) {
|
||||
#if mi_has_builtin(rotateleft32)
|
||||
return mi_builtin(rotateleft32)(x,r);
|
||||
#elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
|
||||
return _lrotl(x, (int)r);
|
||||
#else
|
||||
// The term `(-rshift)&(BITS-1)` is written instead of `BITS - rshift` to
|
||||
// avoid UB when `rshift==0`. See <https://blog.regehr.org/archives/1063>
|
||||
const unsigned int rshift = (unsigned int)(r) & 31;
|
||||
return ((x << rshift) | (x >> ((-rshift) & 31)));
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
#endif // MI_BITS_H
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,542 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#pragma once
|
||||
#ifndef MIMALLOC_PRIM_H
|
||||
#define MIMALLOC_PRIM_H
|
||||
#include "internal.h" // mi_decl_hidden
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// This file specifies the primitive portability API.
|
||||
// Each OS/host needs to implement these primitives, see `src/prim`
|
||||
// for implementations on Window, macOS, WASI, and Linux/Unix.
|
||||
//
|
||||
// note: on all primitive functions, we always have result parameters != NULL, and:
|
||||
// addr != NULL and page aligned
|
||||
// size > 0 and page aligned
|
||||
// the return value is an error code as an `int` where 0 is success
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
// OS memory configuration
|
||||
typedef struct mi_os_mem_config_s {
|
||||
size_t page_size; // default to 4KiB
|
||||
size_t large_page_size; // 0 if not supported, usually 2MiB (4MiB on Windows)
|
||||
size_t alloc_granularity; // smallest allocation size (usually 4KiB, on Windows 64KiB)
|
||||
size_t physical_memory_in_kib; // physical memory size in KiB
|
||||
size_t virtual_address_bits; // usually 48 or 56 bits on 64-bit systems. (used to determine secure randomization)
|
||||
bool has_overcommit; // can we reserve more memory than can be actually committed?
|
||||
bool has_partial_free; // can allocated blocks be freed partially? (true for mmap, false for VirtualAlloc)
|
||||
bool has_virtual_reserve; // supports virtual address space reservation? (if true we can reserve virtual address space without using commit or physical memory)
|
||||
bool has_transparent_huge_pages; // true if transparent huge pages are enabled (on Linux)
|
||||
} mi_os_mem_config_t;
|
||||
|
||||
// Initialize
|
||||
void _mi_prim_mem_init( mi_os_mem_config_t* config );
|
||||
|
||||
// Free OS memory
|
||||
int _mi_prim_free(void* addr, size_t size );
|
||||
|
||||
// Allocate OS memory. Return NULL on error.
|
||||
// The `try_alignment` is just a hint and the returned pointer does not have to be aligned.
|
||||
// If `commit` is false, the virtual memory range only needs to be reserved (with no access)
|
||||
// which will later be committed explicitly using `_mi_prim_commit`.
|
||||
// `is_zero` is set to true if the memory was zero initialized (as on most OS's)
|
||||
// The `hint_addr` address is either `NULL` or a preferred allocation address but can be ignored.
|
||||
// pre: !commit => !allow_large
|
||||
// try_alignment >= _mi_os_page_size() and a power of 2
|
||||
int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr);
|
||||
|
||||
// Commit memory. Returns error code or 0 on success.
|
||||
// For example, on Linux this would make the memory PROT_READ|PROT_WRITE.
|
||||
// `is_zero` is set to true if the memory was zero initialized (e.g. on Windows)
|
||||
int _mi_prim_commit(void* addr, size_t size, bool* is_zero);
|
||||
|
||||
// Decommit memory. Returns error code or 0 on success. The `needs_recommit` result is true
|
||||
// if the memory would need to be re-committed. For example, on Windows this is always true,
|
||||
// but on Linux we could use MADV_DONTNEED to decommit which does not need a recommit.
|
||||
// pre: needs_recommit != NULL
|
||||
int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit);
|
||||
|
||||
// Reset memory. The range keeps being accessible but the content might be reset to zero at any moment.
|
||||
// Returns error code or 0 on success.
|
||||
int _mi_prim_reset(void* addr, size_t size);
|
||||
|
||||
// Reuse memory. This is called for memory that is already committed but
|
||||
// may have been reset (`_mi_prim_reset`) or decommitted (`_mi_prim_decommit`) where `needs_recommit` was false.
|
||||
// Returns error code or 0 on success. On most platforms this is a no-op.
|
||||
int _mi_prim_reuse(void* addr, size_t size);
|
||||
|
||||
// Protect memory. Returns error code or 0 on success.
|
||||
int _mi_prim_protect(void* addr, size_t size, bool protect);
|
||||
|
||||
// Allocate huge (1GiB) pages possibly associated with a NUMA node.
|
||||
// `is_zero` is set to true if the memory was zero initialized (as on most OS's)
|
||||
// pre: size > 0 and a multiple of 1GiB.
|
||||
// numa_node is either negative (don't care), or a numa node number.
|
||||
int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr);
|
||||
|
||||
// Return the current NUMA node
|
||||
size_t _mi_prim_numa_node(void);
|
||||
|
||||
// Return the number of logical NUMA nodes
|
||||
size_t _mi_prim_numa_node_count(void);
|
||||
|
||||
// Clock ticks
|
||||
mi_msecs_t _mi_prim_clock_now(void);
|
||||
|
||||
// Return process information (only for statistics)
|
||||
typedef struct mi_process_info_s {
|
||||
mi_msecs_t elapsed;
|
||||
mi_msecs_t utime;
|
||||
mi_msecs_t stime;
|
||||
size_t current_rss;
|
||||
size_t peak_rss;
|
||||
size_t current_commit;
|
||||
size_t peak_commit;
|
||||
size_t page_faults;
|
||||
} mi_process_info_t;
|
||||
|
||||
void _mi_prim_process_info(mi_process_info_t* pinfo);
|
||||
|
||||
// Default stderr output. (only for warnings etc. with verbose enabled)
|
||||
// msg != NULL && _mi_strlen(msg) > 0
|
||||
void _mi_prim_out_stderr( const char* msg );
|
||||
|
||||
// Get an environment variable. (only for options)
|
||||
// name != NULL, result != NULL, result_size >= 64
|
||||
bool _mi_prim_getenv(const char* name, char* result, size_t result_size);
|
||||
|
||||
|
||||
// Fill a buffer with strong randomness; return `false` on error or if
|
||||
// there is no strong randomization available.
|
||||
bool _mi_prim_random_buf(void* buf, size_t buf_len);
|
||||
|
||||
// Called on the first thread start, and should ensure `_mi_thread_done` is called on thread termination.
|
||||
void _mi_prim_thread_init_auto_done(void);
|
||||
|
||||
// Called on process exit and may take action to clean up resources associated with the thread auto done.
|
||||
void _mi_prim_thread_done_auto_done(void);
|
||||
|
||||
// Called when the default theap for a thread changes
|
||||
void _mi_prim_thread_associate_default_theap(mi_theap_t* theap);
|
||||
|
||||
// Is this thread part of a thread pool?
|
||||
bool _mi_prim_thread_is_in_threadpool(void);
|
||||
|
||||
// Yield to other threads. Should be similar to `sleep(0)`.
|
||||
// Is called only in rare situations and does not have to be lightning fast.
|
||||
void _mi_prim_thread_yield(void);
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Access to TLS (thread local storage) slots.
|
||||
// We need fast access to both a unique thread id (in `free.c:mi_free`) and
|
||||
// to a thread-local theap pointer (in `alloc.c:mi_malloc`).
|
||||
// To achieve this we use specialized code for various platforms.
|
||||
//-------------------------------------------------------------------
|
||||
|
||||
// On some libc + platform combinations we can directly access a thread-local storage (TLS) slot.
|
||||
// The TLS layout depends on both the OS and libc implementation so we use specific tests for each main platform.
|
||||
// If you test on another platform and it works please send a PR :-)
|
||||
// see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register.
|
||||
//
|
||||
// Note: we would like to prefer `__builtin_thread_pointer()` nowadays instead of using assembly,
|
||||
// but unfortunately we can not detect support reliably (see issue #883)
|
||||
// We also use it on Apple OS as we use a TLS slot for the default theap there.
|
||||
#if (defined(_WIN32)) || \
|
||||
(defined(__GNUC__) && ( \
|
||||
(defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \
|
||||
|| (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__) || defined(__POWERPC__))) \
|
||||
|| (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \
|
||||
|| (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \
|
||||
|| (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \
|
||||
))
|
||||
|
||||
static inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept {
|
||||
void* res;
|
||||
const size_t ofs = (slot*sizeof(void*));
|
||||
#if defined(_WIN32)
|
||||
#if (_M_X64 || _M_AMD64) && !defined(_M_ARM64EC)
|
||||
res = (void*)__readgsqword((unsigned long)ofs); // direct load at offset from gs
|
||||
#elif _M_IX86 && !defined(_M_ARM64EC)
|
||||
res = (void*)__readfsdword((unsigned long)ofs); // direct load at offset from fs
|
||||
#else
|
||||
res = ((void**)NtCurrentTeb())[slot]; MI_UNUSED(ofs);
|
||||
#endif
|
||||
#elif defined(__i386__)
|
||||
__asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86 32-bit always uses GS
|
||||
#elif defined(__APPLE__) && defined(__x86_64__)
|
||||
__asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS
|
||||
#elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)
|
||||
__asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI
|
||||
#elif defined(__x86_64__)
|
||||
__asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS
|
||||
#elif defined(__arm__)
|
||||
void** tcb; MI_UNUSED(ofs);
|
||||
__asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
|
||||
res = tcb[slot];
|
||||
#elif defined(__aarch64__)
|
||||
void** tcb; MI_UNUSED(ofs);
|
||||
#if defined(__APPLE__) // M1, issue #343
|
||||
__asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb));
|
||||
#else
|
||||
__asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
|
||||
#endif
|
||||
res = tcb[slot];
|
||||
#elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781
|
||||
MI_UNUSED(ofs);
|
||||
res = pthread_getspecific(slot);
|
||||
#else
|
||||
#define MI_HAS_TLS_SLOT 0
|
||||
MI_UNUSED(ofs);
|
||||
res = NULL;
|
||||
#endif
|
||||
return res;
|
||||
}
|
||||
|
||||
#ifndef MI_HAS_TLS_SLOT
|
||||
#define MI_HAS_TLS_SLOT 1
|
||||
#endif
|
||||
|
||||
// setting a tls slot is only used on macOS for now
|
||||
static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexcept {
|
||||
const size_t ofs = (slot*sizeof(void*));
|
||||
#if defined(_WIN32)
|
||||
((void**)NtCurrentTeb())[slot] = value; MI_UNUSED(ofs);
|
||||
#elif defined(__i386__)
|
||||
__asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS
|
||||
#elif defined(__APPLE__) && defined(__x86_64__)
|
||||
__asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOS uses GS
|
||||
#elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)
|
||||
__asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI
|
||||
#elif defined(__x86_64__)
|
||||
__asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS
|
||||
#elif defined(__arm__)
|
||||
void** tcb; MI_UNUSED(ofs);
|
||||
__asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
|
||||
tcb[slot] = value;
|
||||
#elif defined(__aarch64__)
|
||||
void** tcb; MI_UNUSED(ofs);
|
||||
#if defined(__APPLE__) // M1, issue #343
|
||||
__asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb));
|
||||
#else
|
||||
__asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
|
||||
#endif
|
||||
tcb[slot] = value;
|
||||
#elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781
|
||||
MI_UNUSED(ofs);
|
||||
pthread_setspecific(slot, value);
|
||||
#else
|
||||
MI_UNUSED(ofs); MI_UNUSED(value);
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
// defined in `init.c`; do not use these directly
|
||||
extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_main; // theap belonging to the main heap
|
||||
extern mi_decl_hidden bool _mi_process_is_initialized; // has mi_process_init been called?
|
||||
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Get a fast unique thread id.
|
||||
//
|
||||
// Getting the thread id should be performant as it is called in the
|
||||
// fast path of `_mi_free` and we specialize for various platforms as
|
||||
// inlined definitions. Regular code should call `init.c:_mi_thread_id()`.
|
||||
// We only require _mi_prim_thread_id() to return a unique id
|
||||
// for each thread (unequal to zero).
|
||||
//-------------------------------------------------------------------
|
||||
|
||||
|
||||
// Do we have __builtin_thread_pointer? This would be the preferred way to get a unique thread id
|
||||
// but unfortunately, it seems we cannot test for this reliably at this time (see issue #883)
|
||||
// Nevertheless, it seems needed on older graviton platforms (see issue #851).
|
||||
// For now, we only enable this for specific platforms.
|
||||
#if !defined(MI_USE_BUILTIN_THREAD_POINTER) /* allow user override */
|
||||
#if !defined(__APPLE__) /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly (<https://github.com/microsoft/mimalloc/issues/343#issuecomment-763272369>)*/ \
|
||||
&& !defined(__CYGWIN__) \
|
||||
&& !defined(MI_LIBC_MUSL) \
|
||||
&& (!defined(__clang_major__) || __clang_major__ >= 14) /* older clang versions emit bad code; fall back to using the TLS slot (<https://lore.kernel.org/linux-arm-kernel/202110280952.352F66D8@keescook/T/>) */
|
||||
#if (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \
|
||||
|| (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \
|
||||
|| (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__)))
|
||||
#define MI_USE_BUILTIN_THREAD_POINTER 1
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
|
||||
static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept;
|
||||
|
||||
static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
|
||||
const mi_threadid_t tid = __mi_prim_thread_id();
|
||||
mi_assert_internal(tid > 1);
|
||||
mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0); // bottom 2 bits are clear?
|
||||
return tid;
|
||||
}
|
||||
|
||||
// Get a unique id for the current thread.
|
||||
#if defined(MI_PRIM_THREAD_ID)
|
||||
|
||||
static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
|
||||
const mi_threadid_t tid = MI_PRIM_THREAD_ID(); // used for example by CPython for a free threaded build (see python/cpython#115488)
|
||||
mi_assert_internal( (tid & 0x03) == 0 ); // mimalloc reserves the bottom 2 bits
|
||||
return tid;
|
||||
}
|
||||
|
||||
#elif defined(_WIN32)
|
||||
|
||||
static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
|
||||
// Windows: works on Intel and ARM in both 32- and 64-bit
|
||||
return (uintptr_t)NtCurrentTeb();
|
||||
}
|
||||
|
||||
#elif MI_USE_BUILTIN_THREAD_POINTER
|
||||
|
||||
static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
|
||||
// Works on most Unix based platforms with recent compilers
|
||||
return (uintptr_t)__builtin_thread_pointer();
|
||||
}
|
||||
|
||||
#elif MI_HAS_TLS_SLOT
|
||||
|
||||
static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
|
||||
#if defined(__BIONIC__)
|
||||
// issue #384, #495: on the Bionic libc (Android), slot 1 is the thread id
|
||||
// see: https://github.com/aosp-mirror/platform_bionic/blob/c44b1d0676ded732df4b3b21c5f798eacae93228/libc/platform/bionic/tls_defines.h#L86
|
||||
return (uintptr_t)mi_prim_tls_slot(1);
|
||||
#else
|
||||
// in all our other targets, slot 0 is the thread id
|
||||
// glibc: https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=sysdeps/x86_64/nptl/tls.h
|
||||
// apple: https://github.com/apple/darwin-xnu/blob/main/libsyscall/os/tsd.h#L36
|
||||
return (uintptr_t)mi_prim_tls_slot(0);
|
||||
#endif
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
// otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms).
|
||||
static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
|
||||
return (uintptr_t)&__mi_theap_main;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/* ----------------------------------------------------------------------------------------
|
||||
Get the thread local default theap: `_mi_theap_default()` (and the cached heap `_mi_theap_cached`).
|
||||
|
||||
This is inlined here as it is on the fast path for allocation functions.
|
||||
We have 4 models:
|
||||
|
||||
- MI_TLS_MODEL_THREAD_LOCAL: use regular thread local (default on Linux, FreeBSD, etc)
|
||||
On most platforms (Linux, FreeBSD, NetBSD, etc), this just returns a
|
||||
thread local variable (`__mi_theap_default`). With the initial-exec TLS model this ensures
|
||||
that the storage will always be available and properly initialized (with an empty theap).
|
||||
|
||||
On some platforms the underlying TLS implementation (or the loader) will call itself `malloc`
|
||||
on a first access to a thread local and recurse in the MI_TLS_MODEL_THREAD_LOCAL.
|
||||
A way around this is to define MI_TLS_RECURSE_GUARD which adds an extra check if the process
|
||||
is initialized before accessing the thread-local. This is a check in the fast path though
|
||||
so this should be avoided.
|
||||
|
||||
- MI_TLS_MODEL_FIXED_SLOT: use a fixed slot in the TLS block (default on macOS)
|
||||
This reserves an unused and fixed TLS slot. This is fast and avoids the problem
|
||||
where the underlying TLS implementation (or the loader) will call itself `malloc`
|
||||
on a first access to a thread local (and recurse in the MI_TLS_MODEL_THREAD_LOCAL).
|
||||
This goes wrong though if the OS or a library uses the same fixed slot.
|
||||
|
||||
- MI_TLS_MODEL_DYNAMIC_WIN32: use a dynamically allocated slot with TlsAlloc. (default on Windows)
|
||||
Windows has somewhat slow thread locals so by default we use TlsAlloc'd slots which
|
||||
can be more efficient. First tries to use one of the "direct" first 64 slots which
|
||||
are the fastest, but falls back to using "expansion" slots when needed (up to 1088 slots).
|
||||
(If the allocated slot happens to always be under 64 for a particular program,
|
||||
one might use cmake with `-DMI_WIN_DIRECT_TLS=ON` to skip the expansion slot test in the fast path.)
|
||||
|
||||
- MI_TLS_MODEL_DYNAMIC_PTHREADS: use `pthread_getspecific`. (default on OpenBSD, maybe good for Android as well?)
|
||||
Use pthread local storage. Somewhat slow but can work well depending on the platform.
|
||||
|
||||
Each model should define `MI_THEAP_INITASNULL` to signify that the initial value
|
||||
returned from `_mi_theap_default()` can be `NULL` (instead of the address of the empty heap).
|
||||
This incurs an extra check in the fast path (but can often be combined in an existing check).
|
||||
------------------------------------------------------------------------------------------- */
|
||||
|
||||
static inline mi_theap_t* _mi_theap_default(void);
|
||||
static inline mi_theap_t* _mi_theap_cached(void);
|
||||
|
||||
#if defined(_WIN32)
|
||||
#define MI_TLS_MODEL_DYNAMIC_WIN32 1
|
||||
#elif defined(__APPLE__) && MI_HAS_TLS_SLOT && !defined(__POWERPC__) // macOS on arm64 or x64
|
||||
// #define MI_TLS_MODEL_DYNAMIC_PTHREADS 1 // also works but a bit slower
|
||||
#define MI_TLS_MODEL_FIXED_SLOT 1
|
||||
#define MI_TLS_MODEL_FIXED_SLOT_DEFAULT 108 // seems unused. @apple: it would be great to get 2 official slots for custom allocators :-)
|
||||
#define MI_TLS_MODEL_FIXED_SLOT_CACHED 109
|
||||
// we used before __PTK_FRAMEWORK_OLDGC_KEY9 (89) but that seems used now.
|
||||
// see <https://github.com/rweichler/substrate/blob/master/include/pthread_machdep.h>
|
||||
#elif defined(__APPLE__) || defined(__OpenBSD__) || defined(__ANDROID__)
|
||||
#define MI_TLS_MODEL_DYNAMIC_PTHREADS 1
|
||||
// #define MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL 1
|
||||
#else
|
||||
#define MI_TLS_MODEL_THREAD_LOCAL 1
|
||||
#endif
|
||||
|
||||
// Declared this way to optimize register spills and branches
|
||||
mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void);
|
||||
|
||||
static inline mi_theap_t* __mi_theap_empty(void) {
|
||||
#if __GNUC__
|
||||
__asm(""); // prevent conditional load
|
||||
return (mi_theap_t*)&_mi_theap_empty;
|
||||
#else
|
||||
return _mi_theap_empty_get();
|
||||
#endif
|
||||
}
|
||||
|
||||
#if MI_TLS_MODEL_THREAD_LOCAL
|
||||
// Thread local with an initial value (default on Linux). Very efficient.
|
||||
|
||||
extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default; // default theap to allocate from
|
||||
extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached; // theap from the last used heap
|
||||
|
||||
static inline mi_theap_t* _mi_theap_default(void) {
|
||||
#if defined(MI_TLS_RECURSE_GUARD)
|
||||
if (mi_unlikely(!_mi_process_is_initialized)) return _mi_theap_empty_get();
|
||||
#endif
|
||||
return __mi_theap_default;
|
||||
}
|
||||
|
||||
static inline mi_theap_t* _mi_theap_cached(void) {
|
||||
return __mi_theap_cached;
|
||||
}
|
||||
|
||||
#elif MI_TLS_MODEL_FIXED_SLOT
|
||||
// Fixed TLS slot (default on macOS).
|
||||
#define MI_THEAP_INITASNULL 1
|
||||
|
||||
static inline mi_theap_t* _mi_theap_default(void) {
|
||||
return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_SLOT_DEFAULT);
|
||||
}
|
||||
|
||||
static inline mi_theap_t* _mi_theap_cached(void) {
|
||||
return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_SLOT_CACHED);
|
||||
}
|
||||
|
||||
#elif MI_TLS_MODEL_DYNAMIC_WIN32
|
||||
// Dynamic TLS slot (default on Windows)
|
||||
#define MI_THEAP_INITASNULL 1
|
||||
|
||||
// We try to use direct slots (64), but can also use the expansion slots (upto 1024 extra available)
|
||||
// See <https://www.geoffchappell.com/studies/windows/km/ntoskrnl/inc/api/pebteb/teb/index.htm> for the offsets.
|
||||
#if MI_SIZE_SIZE==4
|
||||
#define MI_TLS_EXPANSION_SLOT (0x0F94 / MI_SIZE_SIZE)
|
||||
#else
|
||||
#define MI_TLS_EXPANSION_SLOT (0x1780 / MI_SIZE_SIZE)
|
||||
#endif
|
||||
|
||||
extern mi_decl_hidden size_t _mi_theap_default_slot;
|
||||
extern mi_decl_hidden size_t _mi_theap_cached_slot;
|
||||
extern mi_decl_hidden size_t _mi_theap_default_expansion_slot;
|
||||
extern mi_decl_hidden size_t _mi_theap_cached_expansion_slot;
|
||||
|
||||
static inline mi_theap_t* _mi_theap_default(void) {
|
||||
const size_t slot = _mi_theap_default_slot;
|
||||
mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot);
|
||||
#if !MI_WIN_DIRECT_TLS
|
||||
if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ?
|
||||
if mi_likely(theap!=NULL) { // initialized (on this thread)?
|
||||
theap = ((mi_theap_t**)theap)[_mi_theap_default_expansion_slot];
|
||||
}
|
||||
}
|
||||
#endif
|
||||
return theap;
|
||||
}
|
||||
|
||||
static inline mi_theap_t* _mi_theap_cached(void) {
|
||||
const size_t slot = _mi_theap_cached_slot;
|
||||
mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot);
|
||||
#if !MI_WIN_DIRECT_TLS
|
||||
if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ?
|
||||
if mi_likely(theap!=NULL) { // initialized (on this thread)?
|
||||
theap = ((mi_theap_t**)theap)[_mi_theap_cached_expansion_slot];
|
||||
}
|
||||
}
|
||||
#endif
|
||||
return theap;
|
||||
}
|
||||
|
||||
#elif MI_TLS_MODEL_DYNAMIC_PTHREADS
|
||||
// Dynamic pthread slot on less common platforms. This is not too bad. (default on OpenBSD)
|
||||
#define MI_THEAP_INITASNULL 1
|
||||
|
||||
extern mi_decl_hidden pthread_key_t _mi_theap_default_key;
|
||||
extern mi_decl_hidden pthread_key_t _mi_theap_cached_key;
|
||||
|
||||
static inline mi_theap_t* _mi_theap_default(void) {
|
||||
#if !MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL
|
||||
// we can skip this check if using the initial key will return NULL from pthread_getspecific
|
||||
if mi_unlikely(_mi_theap_default_key==0) { return NULL; }
|
||||
#endif
|
||||
return (mi_theap_t*)pthread_getspecific(_mi_theap_default_key);
|
||||
}
|
||||
|
||||
static inline mi_theap_t* _mi_theap_cached(void) {
|
||||
#if !MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL
|
||||
// we can skip this check if using the initial key will return NULL from pthread_getspecific
|
||||
if mi_unlikely(_mi_theap_cached_key==0) { return NULL; }
|
||||
#endif
|
||||
return (mi_theap_t*)pthread_getspecific(_mi_theap_cached_key);
|
||||
}
|
||||
|
||||
#else
|
||||
#error "no TLS model is defined for this platform?"
|
||||
#endif
|
||||
|
||||
|
||||
// Check if a thread is initialized (without using a thread-local if using fixed slots)
|
||||
static inline bool _mi_thread_is_initialized(void) {
|
||||
return (mi_theap_is_initialized(_mi_theap_default()));
|
||||
}
|
||||
|
||||
// Get (and possible create) the theap belonging to a heap
|
||||
// We cache the last accessed theap in `_mi_theap_cached` for better performance.
|
||||
static inline mi_theap_t* _mi_heap_theap(const mi_heap_t* heap) {
|
||||
mi_theap_t* theap = _mi_theap_cached();
|
||||
#if MI_THEAP_INITASNULL
|
||||
if mi_likely(theap!=NULL && _mi_theap_heap(theap)==heap) return theap;
|
||||
#else
|
||||
if mi_likely(_mi_theap_heap(theap)==heap) return theap;
|
||||
#endif
|
||||
return _mi_heap_theap_get_or_init(heap);
|
||||
}
|
||||
|
||||
// Get the theap belonging to a heap without creating in if it is not yet initialized.
|
||||
static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap) {
|
||||
mi_theap_t* theap = _mi_theap_cached();
|
||||
#if MI_THEAP_INITASNULL
|
||||
if mi_unlikely(theap==NULL || _mi_theap_heap(theap)!=heap)
|
||||
#else
|
||||
if mi_unlikely(_mi_theap_heap(theap)!=heap)
|
||||
#endif
|
||||
{
|
||||
theap = _mi_heap_theap_get_peek(heap); // don't update the cache on a query (?)
|
||||
}
|
||||
mi_assert(theap==NULL || _mi_theap_heap(theap)==heap);
|
||||
return theap;
|
||||
}
|
||||
|
||||
// Find the associated theap or NULL if it does not exist (during shutdown)
|
||||
// Should be fast as it is called in `free.c:mi_free_try_collect`.
|
||||
static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page) {
|
||||
mi_heap_t* const heap = page->heap;
|
||||
mi_theap_t* theap;
|
||||
if mi_likely(heap==NULL) { theap = __mi_theap_main; } // note: on macOS accessing the thread_local can cause allocation during thread shutdown (and reinitialize the thread)!
|
||||
else { theap = _mi_heap_theap_peek(heap); }
|
||||
mi_assert_internal(theap==NULL || _mi_thread_id()==theap->tld->thread_id);
|
||||
return theap;
|
||||
}
|
||||
|
||||
#endif // MI_PRIM_H
|
||||
@@ -0,0 +1,150 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#pragma once
|
||||
#ifndef MI_TRACK_H
|
||||
#define MI_TRACK_H
|
||||
|
||||
/* ------------------------------------------------------------------------------------------------------
|
||||
Track memory ranges with macros for tools like Valgrind address sanitizer, or other memory checkers.
|
||||
These can be defined for tracking allocation:
|
||||
|
||||
#define mi_track_malloc_size(p,reqsize,size,zero)
|
||||
#define mi_track_free_size(p,_size)
|
||||
|
||||
The macros are set up such that the size passed to `mi_track_free_size`
|
||||
always matches the size of `mi_track_malloc_size`. (currently, `size == mi_usable_size(p)`).
|
||||
The `reqsize` is what the user requested, and `size >= reqsize`.
|
||||
The `size` is either byte precise (and `size==reqsize`) if `MI_PADDING` is enabled,
|
||||
or otherwise it is the usable block size which may be larger than the original request.
|
||||
Use `_mi_block_size_of(void* p)` to get the full block size that was allocated (including padding etc).
|
||||
The `zero` parameter is `true` if the allocated block is zero initialized.
|
||||
|
||||
Optional:
|
||||
|
||||
#define mi_track_align(p,alignedp,offset,size)
|
||||
#define mi_track_resize(p,oldsize,newsize)
|
||||
#define mi_track_init()
|
||||
|
||||
The `mi_track_align` is called right after a `mi_track_malloc` for aligned pointers in a block.
|
||||
The corresponding `mi_track_free` still uses the block start pointer and original size (corresponding to the `mi_track_malloc`).
|
||||
The `mi_track_resize` is currently unused but could be called on reallocations within a block.
|
||||
`mi_track_init` is called at program start.
|
||||
|
||||
The following macros are for tools like asan and valgrind to track whether memory is
|
||||
defined, undefined, or not accessible at all:
|
||||
|
||||
#define mi_track_mem_defined(p,size)
|
||||
#define mi_track_mem_undefined(p,size)
|
||||
#define mi_track_mem_noaccess(p,size)
|
||||
|
||||
-------------------------------------------------------------------------------------------------------*/
|
||||
|
||||
#if MI_TRACK_VALGRIND
|
||||
// valgrind tool
|
||||
|
||||
#define MI_TRACK_ENABLED 1
|
||||
#define MI_TRACK_HEAP_DESTROY 1 // track free of individual blocks on theap_destroy
|
||||
#define MI_TRACK_TOOL "valgrind"
|
||||
|
||||
#include <valgrind/valgrind.h>
|
||||
#include <valgrind/memcheck.h>
|
||||
|
||||
#define mi_track_malloc_size(p,reqsize,size,zero) VALGRIND_MALLOCLIKE_BLOCK(p,size,MI_PADDING_SIZE /*red zone*/,zero)
|
||||
#define mi_track_free_size(p,_size) VALGRIND_FREELIKE_BLOCK(p,MI_PADDING_SIZE /*red zone*/)
|
||||
#define mi_track_resize(p,oldsize,newsize) VALGRIND_RESIZEINPLACE_BLOCK(p,oldsize,newsize,MI_PADDING_SIZE /*red zone*/)
|
||||
#define mi_track_mem_defined(p,size) VALGRIND_MAKE_MEM_DEFINED(p,size)
|
||||
#define mi_track_mem_undefined(p,size) VALGRIND_MAKE_MEM_UNDEFINED(p,size)
|
||||
#define mi_track_mem_noaccess(p,size) VALGRIND_MAKE_MEM_NOACCESS(p,size)
|
||||
|
||||
#elif MI_TRACK_ASAN
|
||||
// address sanitizer
|
||||
|
||||
#define MI_TRACK_ENABLED 1
|
||||
#define MI_TRACK_HEAP_DESTROY 0
|
||||
#define MI_TRACK_TOOL "asan"
|
||||
|
||||
#include <sanitizer/asan_interface.h>
|
||||
|
||||
#define mi_track_malloc_size(p,reqsize,size,zero) ASAN_UNPOISON_MEMORY_REGION(p,size)
|
||||
#define mi_track_free_size(p,size) ASAN_POISON_MEMORY_REGION(p,size)
|
||||
#define mi_track_mem_defined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size)
|
||||
#define mi_track_mem_undefined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size)
|
||||
#define mi_track_mem_noaccess(p,size) ASAN_POISON_MEMORY_REGION(p,size)
|
||||
|
||||
#elif MI_TRACK_ETW
|
||||
// windows event tracing
|
||||
|
||||
#define MI_TRACK_ENABLED 1
|
||||
#define MI_TRACK_HEAP_DESTROY 1
|
||||
#define MI_TRACK_TOOL "ETW"
|
||||
|
||||
#include "../src/prim/windows/etw.h"
|
||||
|
||||
#define mi_track_init() EventRegistermicrosoft_windows_mimalloc()
|
||||
#define mi_track_done() EventUnregistermicrosoft_windows_mimalloc()
|
||||
#define mi_track_malloc_size(p,reqsize,size,zero) EventWriteETW_MI_ALLOC((UINT64)(p), size)
|
||||
#define mi_track_free_size(p,size) EventWriteETW_MI_FREE((UINT64)(p), size)
|
||||
|
||||
#else
|
||||
// no tracking
|
||||
|
||||
#define MI_TRACK_ENABLED 0
|
||||
#define MI_TRACK_HEAP_DESTROY 0
|
||||
#define MI_TRACK_TOOL "none"
|
||||
|
||||
#define mi_track_malloc_size(p,reqsize,size,zero)
|
||||
#define mi_track_free_size(p,_size)
|
||||
|
||||
#endif
|
||||
|
||||
// -------------------
|
||||
// Utility definitions
|
||||
|
||||
#ifndef mi_track_resize
|
||||
#define mi_track_resize(p,oldsize,newsize) do{ mi_track_free_size(p,oldsize); mi_track_malloc(p,newsize,false); } while(0)
|
||||
#endif
|
||||
|
||||
#ifndef mi_track_align
|
||||
#define mi_track_align(p,alignedp,offset,size) mi_track_mem_noaccess(p,offset)
|
||||
#endif
|
||||
|
||||
#ifndef mi_track_init
|
||||
#define mi_track_init()
|
||||
#endif
|
||||
|
||||
#ifndef mi_track_done
|
||||
#define mi_track_done()
|
||||
#endif
|
||||
|
||||
#ifndef mi_track_mem_defined
|
||||
#define mi_track_mem_defined(p,size)
|
||||
#endif
|
||||
|
||||
#ifndef mi_track_mem_undefined
|
||||
#define mi_track_mem_undefined(p,size)
|
||||
#endif
|
||||
|
||||
#ifndef mi_track_mem_noaccess
|
||||
#define mi_track_mem_noaccess(p,size)
|
||||
#endif
|
||||
|
||||
|
||||
#if MI_PADDING
|
||||
#define mi_track_malloc(p,reqsize,zero) \
|
||||
if ((p)!=NULL) { \
|
||||
mi_assert_internal(mi_usable_size(p)==(reqsize)); \
|
||||
mi_track_malloc_size(p,reqsize,reqsize,zero); \
|
||||
}
|
||||
#else
|
||||
#define mi_track_malloc(p,reqsize,zero) \
|
||||
if ((p)!=NULL) { \
|
||||
mi_assert_internal(mi_usable_size(p)>=(reqsize)); \
|
||||
mi_track_malloc_size(p,reqsize,mi_usable_size(p),zero); \
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // MI_TRACK_H
|
||||
@@ -0,0 +1,732 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#pragma once
|
||||
#ifndef MI_TYPES_H
|
||||
#define MI_TYPES_H
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// This file contains the main type definitions for mimalloc:
|
||||
// mi_heap_t : all data for a heap; usually there is just one main default heap.
|
||||
// mi_theap_t : a thread local heap belonging to a specific heap:
|
||||
// maintains lists of thread-local heap pages that have free space.
|
||||
// mi_page_t : a mimalloc page (usually 64KiB or 512KiB) from
|
||||
// where objects of a single size are allocated.
|
||||
// Note: we write "OS page" for OS memory pages while
|
||||
// using plain "page" for mimalloc pages (`mi_page_t`).
|
||||
// mi_arena_t : a large memory area where pages are allocated (process shared)
|
||||
// mi_tld_t : thread local data
|
||||
// mi_subproc_t : all heaps belong to a sub-process (usually just the main one)
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
|
||||
#include <mimalloc-stats.h>
|
||||
#include <stddef.h> // ptrdiff_t
|
||||
#include <stdint.h> // uintptr_t, uint16_t, etc
|
||||
#include <stdbool.h> // bool
|
||||
#include <limits.h> // SIZE_MAX etc.
|
||||
#include <errno.h> // error codes
|
||||
#include "bits.h" // size defines (MI_INTPTR_SIZE etc), bit operations
|
||||
#include "atomic.h" // _Atomic primitives
|
||||
|
||||
// Minimal alignment necessary. On most platforms 16 bytes are needed
|
||||
// due to SSE registers for example. This must be at least `sizeof(void*)`
|
||||
#ifndef MI_MAX_ALIGN_SIZE
|
||||
#define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t)
|
||||
#endif
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Variants
|
||||
// ------------------------------------------------------
|
||||
|
||||
// Define NDEBUG in the release version to disable assertions.
|
||||
// #define NDEBUG
|
||||
|
||||
// Define MI_TRACK_<tool> to enable tracking support
|
||||
// #define MI_TRACK_VALGRIND 1
|
||||
// #define MI_TRACK_ASAN 1
|
||||
// #define MI_TRACK_ETW 1
|
||||
|
||||
// Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance).
|
||||
// #define MI_STAT 1
|
||||
|
||||
// Define MI_SECURE to enable security mitigations
|
||||
// #define MI_SECURE 1 // guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption
|
||||
// #define MI_SECURE 2 // randomize relative allocation addresses (within mimalloc pages)
|
||||
// #define MI_SECURE 3 // encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free)
|
||||
// #define MI_SECURE 4 // checks for double free (may be more expensive) (`-DMI_SECURE=ON`)
|
||||
// #define MI_SECURE 5 // guard page at the end of each mimalloc page (expensive!) (`-DMI_SECURE_FULL=ON`)
|
||||
|
||||
#if !defined(MI_SECURE)
|
||||
#define MI_SECURE 0
|
||||
#endif
|
||||
|
||||
// Define MI_DEBUG for assertion and invariant checking
|
||||
// #define MI_DEBUG 1 // basic assertion checks and statistics, check double free, corrupted free list, and invalid pointer free. (cmake -DMI_DEBUG=ON)
|
||||
// #define MI_DEBUG 2 // + internal assertion checks (cmake -DMI_DEBUG_INTERNAL=ON)
|
||||
// #define MI_DEBUG 3 // + extensive internal invariant checking (cmake -DMI_DEBUG_FULL=ON)
|
||||
#if !defined(MI_DEBUG)
|
||||
#if defined(MI_BUILD_RELEASE) || defined(NDEBUG)
|
||||
#define MI_DEBUG 0
|
||||
#else
|
||||
#define MI_DEBUG 2
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// Statistics (0=only essential, 1=normal, 2=more fine-grained (expensive) tracking)
|
||||
#ifndef MI_STAT
|
||||
#if (MI_DEBUG>0)
|
||||
#define MI_STAT 2
|
||||
#else
|
||||
#define MI_STAT 0
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// Enable guard pages behind objects of a certain size (set by the MIMALLOC_GUARDED_MIN/MAX/SAMPLE_RATE options)
|
||||
#if !defined(MI_GUARDED) && MI_DEBUG && !defined(NDEBUG) && !MI_PAGE_META_ALIGNED_FREE_SMALL
|
||||
#define MI_GUARDED 1
|
||||
#endif
|
||||
|
||||
// Reserve extra padding at the end of each block to be more resilient against theap block overflows.
|
||||
// The padding can detect buffer overflow on free.
|
||||
#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW))
|
||||
#define MI_PADDING 1
|
||||
#endif
|
||||
|
||||
// Check padding bytes; allows byte-precise buffer overflow detection
|
||||
#if !defined(MI_PADDING_CHECK) && MI_PADDING && (MI_SECURE>=3 || MI_DEBUG>=1)
|
||||
#define MI_PADDING_CHECK 1
|
||||
#endif
|
||||
|
||||
|
||||
// Encoded free lists allow detection of corrupted free lists
|
||||
// and can detect buffer overflows, modify after free, and double `free`s.
|
||||
#if (MI_SECURE>=3 || MI_DEBUG>=1)
|
||||
#define MI_ENCODE_FREELIST 1
|
||||
#endif
|
||||
|
||||
// Enable large pages for objects between 64KiB and 512KiB.
|
||||
// This should perhaps be disabled by default as for many workloads the block sizes above 64 KiB
|
||||
// are quite random which can lead to too many partially used large pages (but see issue #1104).
|
||||
#ifndef MI_ENABLE_LARGE_PAGES
|
||||
#define MI_ENABLE_LARGE_PAGES 1
|
||||
#endif
|
||||
|
||||
// Place page meta info at the start of the page area or keep it separate?
|
||||
// Separate keeps the page info at the arena start (default) which is more secure
|
||||
// and reduces wasted space due to alignment and block sizes.
|
||||
// (but also reserves more memory up front (about 2MiB per GiB))
|
||||
#if !defined(MI_PAGE_META_IS_SEPARATED)
|
||||
#if MI_PAGE_MAP_FLAT
|
||||
#define MI_PAGE_META_IS_SEPARATED 0
|
||||
#else
|
||||
#define MI_PAGE_META_IS_SEPARATED 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// We can choose to only put page info of small pages at the start of the page area.
|
||||
// This can be used to have a slightly faster `mi_free_small` function for specialized
|
||||
// cases (like language runtime systems).
|
||||
#if !defined(MI_PAGE_META_ALIGNED_FREE_SMALL)
|
||||
#define MI_PAGE_META_ALIGNED_FREE_SMALL 0
|
||||
#endif
|
||||
|
||||
// Configuration checks
|
||||
#if !MI_PAGE_META_IS_SEPARATED && MI_SECURE
|
||||
#error "secure mode should use separated page infos"
|
||||
#endif
|
||||
#if MI_PAGE_META_ALIGNED_FREE_SMALL && MI_SECURE
|
||||
#error "secure mode cannot use MI_PAGE_META_ALIGNED_FREE_SMALL"
|
||||
#endif
|
||||
#if MI_PAGE_META_IS_SEPARATED && MI_PAGE_MAP_FLAT
|
||||
#error "cannot have a flat page map with separated page infos"
|
||||
#endif
|
||||
#if MI_DEBUG && NDEBUG
|
||||
#warning "mimalloc assertions enabled in a release build"
|
||||
#endif
|
||||
|
||||
|
||||
// --------------------------------------------------------------
|
||||
// Sizes of internal data-structures
|
||||
// (comments specify sizes on 64-bit, usually 32-bit is halved)
|
||||
// --------------------------------------------------------------
|
||||
|
||||
// Main size parameter; determines max arena sizes and max arena object sizes etc.
|
||||
#ifndef MI_ARENA_SLICE_SHIFT
|
||||
#ifdef MI_SMALL_PAGE_SHIFT // backward compatibility
|
||||
#define MI_ARENA_SLICE_SHIFT MI_SMALL_PAGE_SHIFT
|
||||
#elif MI_SECURE>=5 && __APPLE__ && MI_ARCH_ARM64
|
||||
#define MI_ARENA_SLICE_SHIFT (17) // 128 KiB to not waste too much due to 16 KiB guard pages
|
||||
#else
|
||||
#define MI_ARENA_SLICE_SHIFT (13 + MI_SIZE_SHIFT) // 64 KiB (32 KiB on 32-bit)
|
||||
#endif
|
||||
#endif
|
||||
#if MI_ARENA_SLICE_SHIFT < 12
|
||||
#error Arena slices should be at least 4KiB
|
||||
#endif
|
||||
|
||||
#ifndef MI_BCHUNK_BITS_SHIFT
|
||||
#if MI_ARENA_SLICE_SHIFT <= 13 // <= 8KiB
|
||||
#define MI_BCHUNK_BITS_SHIFT (7) // 128 bits
|
||||
#elif MI_ARENA_SLICE_SHIFT < 16 // <= 32KiB
|
||||
#define MI_BCHUNK_BITS_SHIFT (8) // 256 bits
|
||||
#else
|
||||
#define MI_BCHUNK_BITS_SHIFT (6 + MI_SIZE_SHIFT) // 512 bits (or 256 on 32-bit)
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#define MI_BCHUNK_BITS (1 << MI_BCHUNK_BITS_SHIFT) // sub-bitmaps in arena's are "bchunks" of 512 bits
|
||||
#define MI_ARENA_SLICE_SIZE (MI_ZU(1) << MI_ARENA_SLICE_SHIFT) // arena's allocate in slices of 64 KiB
|
||||
#define MI_ARENA_SLICE_ALIGN (MI_ARENA_SLICE_SIZE)
|
||||
|
||||
#define MI_ARENA_MIN_OBJ_SLICES (1)
|
||||
#define MI_ARENA_MAX_CHUNK_OBJ_SLICES (MI_BCHUNK_BITS) // 32 MiB (or 8 MiB on 32-bit)
|
||||
|
||||
#define MI_ARENA_MIN_OBJ_SIZE (MI_ARENA_MIN_OBJ_SLICES * MI_ARENA_SLICE_SIZE)
|
||||
#define MI_ARENA_MAX_CHUNK_OBJ_SIZE (MI_ARENA_MAX_CHUNK_OBJ_SLICES * MI_ARENA_SLICE_SIZE)
|
||||
|
||||
#if MI_ARENA_MAX_CHUNK_OBJ_SIZE < MI_SIZE_SIZE*1024
|
||||
#error maximum object size may be too small to hold local thread data
|
||||
#endif
|
||||
|
||||
#define MI_SMALL_PAGE_SIZE MI_ARENA_MIN_OBJ_SIZE // 64 KiB
|
||||
#define MI_MEDIUM_PAGE_SIZE (8*MI_SMALL_PAGE_SIZE) // 512 KiB (=byte in the bchunk bitmap)
|
||||
#define MI_LARGE_PAGE_SIZE (MI_SIZE_SIZE*MI_MEDIUM_PAGE_SIZE) // 4 MiB (=word in the bchunk bitmap)
|
||||
|
||||
|
||||
// Maximum number of size classes. (spaced exponentially in 12.5% increments)
|
||||
#if MI_BIN_HUGE != 73U
|
||||
#error "mimalloc internal: expecting 73 bins"
|
||||
#endif
|
||||
#define MI_BIN_FULL (MI_BIN_HUGE+1)
|
||||
#define MI_BIN_COUNT (MI_BIN_FULL+1)
|
||||
|
||||
// We never allocate more than PTRDIFF_MAX (see also <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
|
||||
#define MI_MAX_ALLOC_SIZE PTRDIFF_MAX
|
||||
|
||||
// Minimal commit for a page on-demand commit (should be >= OS page size)
|
||||
#define MI_PAGE_MIN_COMMIT_SIZE MI_ARENA_SLICE_SIZE
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Arena's are large reserved areas of memory allocated from
|
||||
// the OS that are managed by mimalloc to efficiently
|
||||
// allocate MI_ARENA_SLICE_SIZE slices of memory for the
|
||||
// mimalloc pages.
|
||||
// ------------------------------------------------------
|
||||
|
||||
// A large memory arena where pages are allocated in.
|
||||
typedef struct mi_arena_s mi_arena_t; // defined below
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Heaps contain allocated blocks. Heaps are self-contained
|
||||
// but share the (sub-process) memory in the arena's.
|
||||
// ------------------------------------------------------
|
||||
|
||||
// A first-class heap.
|
||||
typedef struct mi_heap_s mi_heap_t; // heaps
|
||||
|
||||
// ------------------------------------------------------
|
||||
// We can have sub-processes that are fully separated
|
||||
// from each other (for running multiple Python interpreters
|
||||
// for example). A sub-process holds the memory arenas and heaps.
|
||||
// ------------------------------------------------------
|
||||
|
||||
// A sub-process
|
||||
typedef struct mi_subproc_s mi_subproc_t;
|
||||
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// a memory id tracks the provenance of arena/OS allocated memory
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
// Memory can reside in arena's, direct OS allocated, meta-data pages, or statically allocated.
|
||||
// The memid keeps track of this.
|
||||
typedef enum mi_memkind_e {
|
||||
MI_MEM_NONE, // not allocated
|
||||
MI_MEM_EXTERNAL, // not owned by mimalloc but provided externally (via `mi_manage_os_memory` for example)
|
||||
MI_MEM_STATIC, // allocated in a static area and should not be freed (the initial main theap data for example (`init.c`))
|
||||
MI_MEM_META, // allocated with the meta data allocator (`arena-meta.c`)
|
||||
MI_MEM_OS, // allocated from the OS
|
||||
MI_MEM_OS_HUGE, // allocated as huge OS pages (usually 1GiB, pinned to physical memory)
|
||||
MI_MEM_OS_REMAP, // allocated in a remapable area (i.e. using `mremap`)
|
||||
MI_MEM_ARENA, // allocated from an arena (the usual case) (`arena.c`)
|
||||
MI_MEM_HEAP_MAIN // allocated in the main heap (for theaps)
|
||||
} mi_memkind_t;
|
||||
|
||||
static inline bool mi_memkind_is_os(mi_memkind_t memkind) {
|
||||
return (memkind >= MI_MEM_OS && memkind <= MI_MEM_OS_REMAP);
|
||||
}
|
||||
|
||||
static inline bool mi_memkind_needs_no_free(mi_memkind_t memkind) {
|
||||
return (memkind <= MI_MEM_STATIC);
|
||||
}
|
||||
|
||||
|
||||
typedef struct mi_memid_os_info {
|
||||
void* base; // actual base address of the block (used for offset aligned allocations)
|
||||
size_t size; // allocated full size
|
||||
// size_t alignment; // alignment at allocation
|
||||
} mi_memid_os_info_t;
|
||||
|
||||
typedef struct mi_memid_arena_info {
|
||||
mi_arena_t* arena; // arena that contains this memory
|
||||
uint32_t slice_index; // slice index in the arena
|
||||
uint32_t slice_count; // allocated slices
|
||||
} mi_memid_arena_info_t;
|
||||
|
||||
typedef struct mi_memid_meta_info {
|
||||
void* meta_page; // meta-page that contains the block
|
||||
uint32_t block_index; // block index in the meta-data page
|
||||
uint32_t block_count; // allocated blocks
|
||||
} mi_memid_meta_info_t;
|
||||
|
||||
typedef struct mi_memid_s {
|
||||
union {
|
||||
mi_memid_os_info_t os; // only used for MI_MEM_OS
|
||||
mi_memid_arena_info_t arena; // only used for MI_MEM_ARENA
|
||||
mi_memid_meta_info_t meta; // only used for MI_MEM_META
|
||||
} mem;
|
||||
mi_memkind_t memkind;
|
||||
bool is_pinned; // `true` if we cannot decommit/reset/protect in this memory (e.g. when allocated using large (2Mib) or huge (1GiB) OS pages)
|
||||
bool initially_committed;// `true` if the memory was originally allocated as committed
|
||||
bool initially_zero; // `true` if the memory was originally zero initialized
|
||||
} mi_memid_t;
|
||||
|
||||
|
||||
static inline bool mi_memid_is_os(mi_memid_t memid) {
|
||||
return mi_memkind_is_os(memid.memkind);
|
||||
}
|
||||
|
||||
static inline bool mi_memid_needs_no_free(mi_memid_t memid) {
|
||||
return mi_memkind_needs_no_free(memid.memkind);
|
||||
}
|
||||
|
||||
static inline mi_arena_t* mi_memid_arena(mi_memid_t memid) {
|
||||
return (memid.memkind == MI_MEM_ARENA ? memid.mem.arena.arena : NULL);
|
||||
}
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Mimalloc pages contain allocated blocks
|
||||
// ------------------------------------------------------
|
||||
|
||||
// The free lists use encoded next fields
|
||||
// (Only actually encodes when MI_ENCODED_FREELIST is defined.)
|
||||
typedef uintptr_t mi_encoded_t;
|
||||
|
||||
// thread id's
|
||||
typedef size_t mi_threadid_t;
|
||||
|
||||
// free lists contain blocks
|
||||
typedef struct mi_block_s {
|
||||
mi_encoded_t next;
|
||||
} mi_block_t;
|
||||
|
||||
|
||||
// The page flags are put in the bottom 2 bits of the thread_id (for a fast test in `mi_free`)
|
||||
// If `has_interior_pointers` is true if the page has pointers at an offset in a block (so we have to unalign to the block start before free-ing)
|
||||
// `in_full_queue` is true if the page is full and resides in the full queue (so we move it to a regular queue on free-ing)
|
||||
#define MI_PAGE_IN_FULL_QUEUE MI_ZU(0x01)
|
||||
#define MI_PAGE_HAS_INTERIOR_POINTERS MI_ZU(0x02)
|
||||
#define MI_PAGE_FLAG_MASK MI_ZU(0x03)
|
||||
typedef size_t mi_page_flags_t;
|
||||
|
||||
// There are two special threadid's: 0 for pages that are abandoned (and not in a theap queue),
|
||||
// and 4 for abandoned & mapped threads -- abandoned-mapped pages are abandoned but also mapped
|
||||
// in an arena (in `mi_heap_t.arena_pages.pages_abandoned`) so these can be quickly found for reuse.
|
||||
// Abandoning partially used pages allows for sharing of this memory between threads (in particular if threads are blocked)
|
||||
#define MI_THREADID_ABANDONED MI_ZU(0)
|
||||
#define MI_THREADID_ABANDONED_MAPPED (MI_PAGE_FLAG_MASK + 1)
|
||||
|
||||
// Thread free list.
|
||||
// Points to a list of blocks that are freed by other threads.
|
||||
// The least-bit is set if the page is owned by the current thread. (`mi_page_is_owned`).
|
||||
// Ownership is required before we can read any non-atomic fields in the page.
|
||||
// This way we can push a block on the thread free list and try to claim ownership atomically in `free.c:mi_free_block_mt`.
|
||||
typedef uintptr_t mi_thread_free_t;
|
||||
|
||||
// A page contains blocks of one specific size (`block_size`).
|
||||
// Each page has three list of free blocks:
|
||||
// `free` for blocks that can be allocated,
|
||||
// `local_free` for freed blocks that are not yet available to `mi_malloc`
|
||||
// `thread_free` for freed blocks by other threads
|
||||
// The `local_free` and `thread_free` lists are migrated to the `free` list
|
||||
// when it is exhausted. The separate `local_free` list is necessary to
|
||||
// implement a monotonic heartbeat. The `thread_free` list is needed for
|
||||
// avoiding atomic operations when allocating from the owning thread.
|
||||
//
|
||||
// `used - |thread_free|` == actual blocks that are in use (alive)
|
||||
// `used - |thread_free| + |free| + |local_free| == capacity`
|
||||
//
|
||||
// We don't count "freed" (as |free|) but use only the `used` field to reduce
|
||||
// the number of memory accesses in the `mi_page_all_free` function(s).
|
||||
// Use `_mi_page_free_collect` to collect the thread_free list and update the `used` count.
|
||||
//
|
||||
// Notes:
|
||||
// - Non-atomic fields can only be accessed if having _ownership_ (low bit of `xthread_free` is 1).
|
||||
// Combining the `thread_free` list with an ownership bit allows a concurrent `free` to atomically
|
||||
// free an object and (re)claim ownership if the page was abandoned.
|
||||
// - If a page is not part of a theap it is called "abandoned" (`theap==NULL`) -- in
|
||||
// that case the `xthreadid` is 0 or 4 (4 is for abandoned pages that
|
||||
// are in the `pages_abandoned` lists of an arena, these are called "mapped" abandoned pages).
|
||||
// - page flags are in the bottom 3 bits of `xthread_id` for the fast path in `mi_free`.
|
||||
// - The layout is optimized for `free.c:mi_free` and `alloc.c:mi_page_alloc`
|
||||
// - Using `uint16_t` does not seem to slow things down
|
||||
|
||||
typedef struct mi_page_s {
|
||||
_Atomic(mi_threadid_t) xthread_id; // thread this page belongs to. (= `theap->thread_id (or 0 or 4 if abandoned) | page_flags`)
|
||||
|
||||
mi_block_t* free; // list of available free blocks (`malloc` allocates from this list)
|
||||
uint16_t used; // number of blocks in use (including blocks in `thread_free`)
|
||||
uint16_t capacity; // number of blocks committed
|
||||
uint16_t reserved; // number of blocks reserved in memory
|
||||
uint8_t retire_expire; // expiration count for retired blocks
|
||||
bool free_is_zero; // `true` if the blocks in the free list are zero initialized
|
||||
|
||||
mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`)
|
||||
_Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads (= `mi_block_t* | (1 if owned)`)
|
||||
|
||||
size_t block_size; // const: size available in each block (always `>0`)
|
||||
uint8_t* page_start; // const: start of the blocks
|
||||
|
||||
#if (MI_ENCODE_FREELIST || MI_PADDING)
|
||||
uintptr_t keys[2]; // const: two random keys to encode the free lists (see `_mi_block_next`) or padding canary
|
||||
#endif
|
||||
|
||||
mi_theap_t* theap; // the theap owning this page (may not be valid or NULL for abandoned pages)
|
||||
mi_heap_t* heap; // const: the heap owning this page
|
||||
|
||||
struct mi_page_s* next; // next page owned by the theap with the same `block_size`
|
||||
struct mi_page_s* prev; // previous page owned by the theap with the same `block_size`
|
||||
size_t slice_committed; // committed size relative to the first arena slice of the page data (or 0 if the page is fully committed already)
|
||||
mi_memid_t memid; // const: provenance of the page memory
|
||||
} mi_page_t;
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Object sizes
|
||||
// ------------------------------------------------------
|
||||
|
||||
#define MI_PAGE_ALIGN MI_ARENA_SLICE_ALIGN // pages must be aligned on this for the page map.
|
||||
#define MI_PAGE_MIN_START_BLOCK_ALIGN MI_MAX_ALIGN_SIZE // minimal block alignment for the first block in a page (16b)
|
||||
#define MI_PAGE_MAX_START_BLOCK_ALIGN2 (4*MI_KiB) // maximal block alignment for "power of 2"-sized blocks (such that we guarantee natural alignment)
|
||||
#define MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4*MI_KiB) // also aligns any multiple of this size to avoid TLB misses.
|
||||
#define MI_PAGE_MAX_OVERALLOC_ALIGN MI_ARENA_SLICE_SIZE // (64 KiB) limit for which we overallocate in arena pages, beyond this use OS allocation
|
||||
|
||||
// The max object sizes are intended to not waste more than ~ 12.5% internally over the page sizes.
|
||||
#define MI_SMALL_MAX_OBJ_SIZE ((MI_SMALL_PAGE_SIZE-MI_PAGE_OSPAGE_BLOCK_ALIGN2)/6) // = 10 KiB
|
||||
#if MI_ENABLE_LARGE_PAGES
|
||||
#define MI_MEDIUM_MAX_OBJ_SIZE ((MI_MEDIUM_PAGE_SIZE-MI_PAGE_OSPAGE_BLOCK_ALIGN2)/6) // ~ 84 KiB
|
||||
#define MI_LARGE_MAX_OBJ_SIZE (MI_LARGE_PAGE_SIZE/8) // <= 512 KiB // note: this must be a nice power of 2 or we get rounding issues with `_mi_bin`
|
||||
#else
|
||||
#define MI_MEDIUM_MAX_OBJ_SIZE (MI_MEDIUM_PAGE_SIZE/8) // <= 64 KiB
|
||||
#define MI_LARGE_MAX_OBJ_SIZE MI_MEDIUM_MAX_OBJ_SIZE // note: this must be a nice power of 2 or we get rounding issues with `_mi_bin`
|
||||
#endif
|
||||
#define MI_LARGE_MAX_OBJ_WSIZE (MI_LARGE_MAX_OBJ_SIZE/MI_SIZE_SIZE)
|
||||
|
||||
#if (MI_LARGE_MAX_OBJ_WSIZE >= 655360)
|
||||
#error "mimalloc internal: define more bins"
|
||||
#endif
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Page kinds
|
||||
// ------------------------------------------------------
|
||||
|
||||
typedef enum mi_page_kind_e {
|
||||
MI_PAGE_SMALL, // small blocks go into 64KiB pages
|
||||
MI_PAGE_MEDIUM, // medium blocks go into 512KiB pages
|
||||
MI_PAGE_LARGE, // larger blocks go into 4MiB pages (if `MI_ENABLE_LARGE_PAGES==1`)
|
||||
MI_PAGE_SINGLETON // page containing a single block.
|
||||
// used for blocks `> MI_LARGE_MAX_OBJ_SIZE` or an alignment `> MI_PAGE_MAX_OVERALLOC_ALIGN`.
|
||||
} mi_page_kind_t;
|
||||
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// A "theap" is a thread local heap which owns pages.
|
||||
// (making them thread-local avoids atomic operations)
|
||||
//
|
||||
// All theaps belong to a (non-thread-local) heap.
|
||||
// A theap just owns a set of pages for allocation and
|
||||
// can only be allocate/reallocate from the thread that created it.
|
||||
// Freeing blocks can be done from any thread though.
|
||||
//
|
||||
// Per thread, there is always a default theap that belongs
|
||||
// to the default heap. It is initialized to statically
|
||||
// point initially to an empty theap to avoid initialization
|
||||
// checks in the fast path.
|
||||
// ------------------------------------------------------
|
||||
|
||||
// Thread local data
|
||||
typedef struct mi_tld_s mi_tld_t; // defined below
|
||||
|
||||
// Pages of a certain block size are held in a queue.
|
||||
typedef struct mi_page_queue_s {
|
||||
mi_page_t* first;
|
||||
mi_page_t* last;
|
||||
size_t count;
|
||||
size_t block_size;
|
||||
} mi_page_queue_t;
|
||||
|
||||
// Random context
|
||||
typedef struct mi_random_cxt_s {
|
||||
uint32_t input[16];
|
||||
uint32_t output[16];
|
||||
int output_available;
|
||||
bool weak;
|
||||
} mi_random_ctx_t;
|
||||
|
||||
|
||||
// In debug mode there is a padding structure at the end of the blocks to check for buffer overflows
|
||||
#if MI_PADDING
|
||||
typedef struct mi_padding_s {
|
||||
uint32_t canary; // encoded block value to check validity of the padding (in case of overflow)
|
||||
uint32_t delta; // padding bytes before the block. (mi_usable_size(p) - delta == exact allocated bytes)
|
||||
} mi_padding_t;
|
||||
#define MI_PADDING_SIZE (sizeof(mi_padding_t))
|
||||
#define MI_PADDING_WSIZE ((MI_PADDING_SIZE + MI_INTPTR_SIZE - 1) / MI_INTPTR_SIZE)
|
||||
#else
|
||||
#define MI_PADDING_SIZE 0
|
||||
#define MI_PADDING_WSIZE 0
|
||||
#endif
|
||||
|
||||
#define MI_PAGES_DIRECT (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE + 1)
|
||||
|
||||
|
||||
// A thread-local heap ("theap") owns a set of thread-local pages.
|
||||
struct mi_theap_s {
|
||||
mi_tld_t* tld; // thread-local data
|
||||
_Atomic(mi_heap_t*) heap; // the heap this theap belongs to.
|
||||
_Atomic(size_t) refcount; // reference count
|
||||
unsigned long long heartbeat; // monotonic heartbeat count
|
||||
uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`)
|
||||
mi_random_ctx_t random; // random number context used for secure allocation
|
||||
size_t page_count; // total number of pages in the `pages` queues.
|
||||
size_t page_retired_min; // smallest retired index (retired pages are fully free, but still in the page queues)
|
||||
size_t page_retired_max; // largest retired index into the `pages` array.
|
||||
size_t pages_full_size; // optimization: total size of blocks in the pages of the full queue (issue #1220)
|
||||
long generic_count; // how often is `_mi_malloc_generic` called?
|
||||
long generic_collect_count; // how often is `_mi_malloc_generic` called without collecting?
|
||||
|
||||
mi_theap_t* tnext; // list of theaps in this thread
|
||||
mi_theap_t* tprev;
|
||||
mi_theap_t* hnext; // list of theaps of the owning `heap`
|
||||
mi_theap_t* hprev;
|
||||
|
||||
long page_full_retain; // how many full pages can be retained per queue (before abandoning them)
|
||||
bool allow_page_reclaim; // `true` if this theap should not reclaim abandoned pages
|
||||
bool allow_page_abandon; // `true` if this theap can abandon pages to reduce memory footprint
|
||||
#if MI_GUARDED
|
||||
size_t guarded_size_min; // minimal size for guarded objects
|
||||
size_t guarded_size_max; // maximal size for guarded objects
|
||||
size_t guarded_sample_rate; // sample rate (set to 0 to disable guarded pages)
|
||||
size_t guarded_sample_count; // current sample count (counting down to 0)
|
||||
#endif
|
||||
mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size.
|
||||
mi_page_queue_t pages[MI_BIN_COUNT]; // queue of pages for each size class (or "bin")
|
||||
mi_memid_t memid; // provenance of the theap struct itself (meta or os)
|
||||
mi_stats_t stats; // thread-local statistics
|
||||
};
|
||||
|
||||
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Heaps contain allocated blocks. Heaps are self-contained
|
||||
// but share the (sub-process) memory in the arena's.
|
||||
// ------------------------------------------------------
|
||||
|
||||
// Keep track of all owned and abandoned pages in the arena's
|
||||
struct mi_arena_pages_s;
|
||||
typedef struct mi_arena_pages_s mi_arena_pages_t;
|
||||
|
||||
#define MI_MAX_ARENAS (160) // Limited for now (and takes up .bss).. but arena's scale up exponentially (see `mi_arena_reserve`)
|
||||
// 160 arenas is enough for ~2 TiB memory
|
||||
|
||||
// A dynamic thread-local variable; 0 for an invalid thread-local
|
||||
typedef size_t mi_thread_local_t;
|
||||
|
||||
typedef struct mi_heap_s {
|
||||
mi_subproc_t* subproc; // a heap belongs to a subprocess
|
||||
size_t heap_seq; // unique sequence number for heaps in this subprocess
|
||||
mi_heap_t* next; // list of heaps in this subprocess
|
||||
mi_heap_t* prev;
|
||||
mi_thread_local_t theap; // dynamic thread local for the thread-local theaps of this heap
|
||||
|
||||
mi_arena_t* exclusive_arena; // if the heap should only allocate from a specific arena (or NULL)
|
||||
int numa_node; // if >=0, prefer this numa node for allocations
|
||||
|
||||
mi_theap_t* theaps; // list of all thread-local theaps belonging to this heap (using the `hnext`/`hprev` fields)
|
||||
mi_lock_t theaps_lock; // lock for the theaps list operations
|
||||
|
||||
_Atomic(size_t) abandoned_count[MI_BIN_COUNT]; // total count of abandoned pages in this heap
|
||||
mi_page_t* os_abandoned_pages; // list of pages that are OS allocated and not in an arena
|
||||
mi_lock_t os_abandoned_pages_lock; // lock for the os abandoned pages list (this lock protects list operations)
|
||||
|
||||
_Atomic(mi_arena_pages_t*) arena_pages[MI_MAX_ARENAS]; // track owned and abandoned pages in the arenas (entries can be NULL)
|
||||
mi_lock_t arena_pages_lock; // lock to update the arena_pages array
|
||||
|
||||
mi_stats_t stats; // statistics for this heap; periodically updated by merging from each theap
|
||||
} mi_heap_t;
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Sub processes do not reclaim or visit pages from other sub processes.
|
||||
// These are essentially the static variables of a process, and
|
||||
// usually there is only one subprocess. This can be used for example
|
||||
// by CPython to have separate interpreters within one process.
|
||||
// Each thread can only belong to one subprocess
|
||||
// (and needs to call `mi_subproc_add_current_thread` before any allocations).
|
||||
// ------------------------------------------------------
|
||||
|
||||
struct mi_subproc_s {
|
||||
size_t subproc_seq; // unique id for sub-processes
|
||||
mi_subproc_t* next; // list of all sub-processes
|
||||
mi_subproc_t* prev;
|
||||
|
||||
_Atomic(size_t) arena_count; // current count of arena's
|
||||
_Atomic(mi_arena_t*) arenas[MI_MAX_ARENAS]; // arena's of this sub-process
|
||||
mi_lock_t arena_reserve_lock; // lock to ensure arena's get reserved one at a time
|
||||
mi_decl_align(8) // needed on some 32-bit platforms
|
||||
_Atomic(int64_t) purge_expire; // expiration is set if any arenas can be purged
|
||||
|
||||
_Atomic(mi_heap_t*) heap_main; // main heap for this sub process
|
||||
mi_heap_t* heaps; // heaps belonging to this sub-process
|
||||
mi_lock_t heaps_lock;
|
||||
|
||||
_Atomic(size_t) thread_count; // current threads associated with this sub-process
|
||||
_Atomic(size_t) thread_total_count; // total created threads associated with this sub-process
|
||||
_Atomic(size_t) heap_count; // current heaps in this sub-process (== |heaps|)
|
||||
_Atomic(size_t) heap_total_count; // total created heaps in this sub-process
|
||||
|
||||
mi_memid_t memid; // provenance of this memory block (meta or static)
|
||||
mi_decl_align(8) // needed on some 32-bit platforms
|
||||
mi_stats_t stats; // subprocess statistics; updated for arena/OS stats like committed,
|
||||
// and otherwise merged with heap stats when those are deleted
|
||||
};
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Thread Local data
|
||||
// ------------------------------------------------------
|
||||
|
||||
// Milliseconds as in `int64_t` to avoid overflows
|
||||
typedef int64_t mi_msecs_t;
|
||||
|
||||
// Thread local data
|
||||
struct mi_tld_s {
|
||||
mi_threadid_t thread_id; // thread id of this thread
|
||||
size_t thread_seq; // thread sequence id (linear count of created threads)
|
||||
int numa_node; // thread preferred numa node
|
||||
mi_subproc_t* subproc; // sub-process this thread belongs to.
|
||||
mi_theap_t* theaps; // list of theaps in this thread (so we can abandon all when the thread terminates)
|
||||
mi_lock_t theaps_lock; // lock as the theaps list is sometimes accessed from another thread (on `mi_heap_free`)
|
||||
bool recurse; // true if deferred was called; used to prevent infinite recursion.
|
||||
bool is_in_threadpool; // true if this thread is part of a threadpool (and can run arbitrary tasks)
|
||||
mi_memid_t memid; // provenance of the tld memory itself (meta or OS)
|
||||
};
|
||||
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
Arenas are fixed area's of OS memory from which we can allocate
|
||||
large blocks (>= MI_ARENA_MIN_BLOCK_SIZE).
|
||||
In contrast to the rest of mimalloc, the arenas are shared between
|
||||
threads and need to be accessed using atomic operations (using atomic `mi_bitmap_t`'s).
|
||||
|
||||
Arenas are also used to for huge OS page (1GiB) reservations or for reserving
|
||||
OS memory upfront which can be improve performance or is sometimes needed
|
||||
on embedded devices. We can also employ this with WASI or `sbrk` systems
|
||||
to reserve large arenas upfront and be able to reuse the memory more effectively.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
#define MI_ARENA_BIN_COUNT (MI_BIN_COUNT)
|
||||
#define MI_ARENA_MIN_SIZE (MI_BCHUNK_BITS * MI_ARENA_SLICE_SIZE) // 32 MiB (or 8 MiB on 32-bit)
|
||||
#define MI_ARENA_MAX_SIZE (MI_BITMAP_MAX_BIT_COUNT * MI_ARENA_SLICE_SIZE)
|
||||
|
||||
typedef struct mi_bitmap_s mi_bitmap_t; // atomic bitmap (defined in `src/bitmap.h`)
|
||||
typedef struct mi_bbitmap_s mi_bbitmap_t; // atomic binned bitmap (defined in `src/bitmap.h`)
|
||||
|
||||
typedef struct mi_arena_pages_s {
|
||||
mi_bitmap_t* pages; // all registered pages (abandoned and owned)
|
||||
mi_bitmap_t* pages_abandoned[MI_ARENA_BIN_COUNT]; // abandoned pages per size bin (a set bit means the start of the page)
|
||||
// followed by the bitmaps (whose siz`es depend on the arena size)
|
||||
} mi_arena_pages_t;
|
||||
|
||||
|
||||
// A memory arena
|
||||
typedef struct mi_arena_s {
|
||||
mi_memid_t memid; // provenance of the memory area
|
||||
mi_subproc_t* subproc; // subprocess this arena belongs to (`this 'element-of' this->subproc->arenas`)
|
||||
size_t arena_idx; // index in the arenas array
|
||||
|
||||
size_t slice_count; // total size of the area in arena slices (of `MI_ARENA_SLICE_SIZE`)
|
||||
size_t info_slices; // initial slices reserved for the arena bitmaps
|
||||
int numa_node; // associated NUMA node
|
||||
bool is_exclusive; // only allow allocations if specifically for this arena
|
||||
mi_decl_align(8) // needed on some 32-bit platforms
|
||||
_Atomic(mi_msecs_t) purge_expire; // expiration time when slices can be purged from `slices_purge`.
|
||||
mi_commit_fun_t* commit_fun; // custom commit/decommit memory
|
||||
void* commit_fun_arg; // user argument for a custom commit function
|
||||
|
||||
size_t total_size; // for (user given) memory more than MI_ARENA_MAX_SIZE, we use N arena's to cover it. The first (parent) has the total size (and the other sub-arena's 0).
|
||||
mi_arena_t* parent; // if this is a sub arena, this points to the first one in the memory area.
|
||||
|
||||
mi_bbitmap_t* slices_free; // is the slice free? (a binned bitmap with size classes)
|
||||
mi_bitmap_t* slices_committed; // is the slice committed? (i.e. accessible)
|
||||
mi_bitmap_t* slices_dirty; // is the slice potentially non-zero?
|
||||
mi_bitmap_t* slices_purge; // slices that can be purged
|
||||
mi_page_t* pages_meta; // pre-allocated `slice_count` page meta info -- only used if `MI_PAGE_META_IS_SEPARATED!=0`
|
||||
mi_arena_pages_t pages_main; // arena page bitmaps for the main heap are allocated up front as well
|
||||
|
||||
// followed by the bitmaps (whose sizes depend on the arena size)
|
||||
// note: when adding bitmaps revise `mi_arena_info_slices_needed`
|
||||
} mi_arena_t;
|
||||
|
||||
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Error codes passed to `_mi_fatal_error`
|
||||
All are recoverable but EFAULT is a serious error and aborts by default in secure mode.
|
||||
For portability define undefined error codes using common Unix codes:
|
||||
<https://www-numi.fnal.gov/offline_software/srt_public_context/WebDocs/Errors/unix_system_errors.html>
|
||||
----------------------------------------------------------- */
|
||||
|
||||
#ifndef EAGAIN // double free
|
||||
#define EAGAIN (11)
|
||||
#endif
|
||||
#ifndef ENOMEM // out of memory
|
||||
#define ENOMEM (12)
|
||||
#endif
|
||||
#ifndef EFAULT // corrupted free-list or meta-data
|
||||
#define EFAULT (14)
|
||||
#endif
|
||||
#ifndef EINVAL // trying to free an invalid pointer
|
||||
#define EINVAL (22)
|
||||
#endif
|
||||
#ifndef EOVERFLOW // count*size overflow
|
||||
#define EOVERFLOW (75)
|
||||
#endif
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Debug constants
|
||||
----------------------------------------------------------- */
|
||||
|
||||
#if !defined(MI_DEBUG_UNINIT)
|
||||
#define MI_DEBUG_UNINIT (0xD0)
|
||||
#endif
|
||||
#if !defined(MI_DEBUG_FREED)
|
||||
#define MI_DEBUG_FREED (0xDF)
|
||||
#endif
|
||||
#if !defined(MI_DEBUG_PADDING)
|
||||
#define MI_DEBUG_PADDING (0xDE)
|
||||
#endif
|
||||
|
||||
|
||||
#endif // MI_TYPES_H
|
||||
@@ -0,0 +1,441 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc/internal.h"
|
||||
#include "mimalloc/prim.h" // _mi_theap_default
|
||||
|
||||
#include <string.h> // memset
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Aligned Allocation
|
||||
// ------------------------------------------------------
|
||||
|
||||
static bool mi_malloc_is_naturally_aligned( size_t size, size_t alignment ) {
|
||||
// certain blocks are always allocated at a certain natural alignment.
|
||||
// (see also `arena.c:mi_arenas_page_alloc_fresh`).
|
||||
mi_assert_internal(_mi_is_power_of_two(alignment) && (alignment > 0));
|
||||
if (alignment > size) return false;
|
||||
const size_t bsize = mi_good_size(size);
|
||||
const bool ok = (bsize <= MI_PAGE_MAX_START_BLOCK_ALIGN2 && _mi_is_power_of_two(bsize)) || // power-of-two under N
|
||||
(alignment==MI_PAGE_OSPAGE_BLOCK_ALIGN2 && (bsize % MI_PAGE_OSPAGE_BLOCK_ALIGN2)==0); // or multiple of N
|
||||
if (ok) { mi_assert_internal((bsize & (alignment-1)) == 0); } // since both power of 2 and alignment <= size
|
||||
return ok;
|
||||
}
|
||||
|
||||
#if MI_GUARDED
|
||||
static mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap, size_t size, size_t alignment, bool zero) mi_attr_noexcept {
|
||||
// use over allocation for guarded blocksl
|
||||
#if MI_THEAP_INITASNULL
|
||||
if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); }
|
||||
#endif
|
||||
mi_assert_internal(alignment > 0 && alignment < MI_PAGE_MAX_OVERALLOC_ALIGN);
|
||||
if mi_unlikely(alignment >= MI_PAGE_MAX_OVERALLOC_ALIGN || size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE - alignment)) {
|
||||
_mi_error_message(EOVERFLOW, "(guarded) aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment);
|
||||
return NULL;
|
||||
}
|
||||
const size_t oversize = size + alignment - 1;
|
||||
void* const base = _mi_theap_malloc_guarded(theap, oversize, zero);
|
||||
if (base==NULL) return NULL;
|
||||
void* const p = _mi_align_up_ptr(base, alignment);
|
||||
mi_track_align(base, p, (uint8_t*)p - (uint8_t*)base, size);
|
||||
mi_assert_internal(mi_usable_size(p) >= size);
|
||||
mi_assert_internal(_mi_is_aligned(p, alignment));
|
||||
return p;
|
||||
}
|
||||
|
||||
static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, size_t* usable) {
|
||||
#if MI_THEAP_INITASNULL
|
||||
if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); }
|
||||
#endif
|
||||
const size_t rate = theap->guarded_sample_rate;
|
||||
// only write if `rate!=0` so we don't write to the constant `_mi_theap_empty`
|
||||
if (rate != 0) { theap->guarded_sample_rate = 0; }
|
||||
void* p = _mi_theap_malloc_zero(theap, size, zero, usable);
|
||||
if (rate != 0) { theap->guarded_sample_rate = rate; }
|
||||
return p;
|
||||
}
|
||||
#else
|
||||
static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, size_t* usable) {
|
||||
return _mi_theap_malloc_zero(theap, size, zero, usable);
|
||||
}
|
||||
#endif
|
||||
|
||||
// Fallback aligned allocation that over-allocates -- split out for better codegen
|
||||
static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept
|
||||
{
|
||||
mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE));
|
||||
mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment));
|
||||
|
||||
void* p;
|
||||
size_t oversize;
|
||||
if mi_unlikely(alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) {
|
||||
// use OS allocation for large alignments and allocate inside a singleton page (not in an arena)
|
||||
// This can support alignments >= MI_PAGE_ALIGN by ensuring the object can be aligned
|
||||
// in the first (and single) page such that the page info is `MI_PAGE_ALIGN` bytes before it (and can be found in the _mi_page_map).
|
||||
if mi_unlikely(offset != 0) {
|
||||
// todo: cannot support offset alignment for very large alignments yet
|
||||
_mi_error_message(EOVERFLOW, "aligned allocation with a large alignment cannot be used with an alignment offset (size %zu, alignment %zu, offset %zu)\n", size, alignment, offset);
|
||||
return NULL;
|
||||
}
|
||||
oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size);
|
||||
// note: no guarded as alignment > 0
|
||||
p = _mi_theap_malloc_zero_ex(theap, oversize, zero, alignment, usable); // the page block size should be large enough to align in the single huge page block
|
||||
if (p == NULL) return NULL;
|
||||
}
|
||||
else {
|
||||
// otherwise over-allocate
|
||||
mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) && alignment <= MI_PAGE_MAX_OVERALLOC_ALIGN);
|
||||
mi_assert_internal(size < SIZE_MAX - alignment); // `oversize` cannot overflow
|
||||
oversize = (size < MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : size) + alignment - 1; // adjust for size <= 16; with size 0 and alignment 64k, we would allocate a 64k block and pointing just beyond that.
|
||||
p = mi_theap_malloc_zero_no_guarded(theap, oversize, zero, usable);
|
||||
if (p == NULL) return NULL;
|
||||
}
|
||||
|
||||
// .. and align within the allocation
|
||||
const uintptr_t align_mask = alignment - 1; // for any x, `(x & align_mask) == (x % alignment)`
|
||||
const uintptr_t poffset = ((uintptr_t)p + offset) & align_mask;
|
||||
const uintptr_t adjust = (poffset == 0 ? 0 : alignment - poffset);
|
||||
mi_assert_internal(adjust < alignment);
|
||||
void* aligned_p = (void*)((uintptr_t)p + adjust);
|
||||
|
||||
// note: after the above allocation, the page may be abandoned now (as it became full, see `page.c:_mi_malloc_generic`)
|
||||
// and we no longer own it. We should be careful to only read constant fields in the page,
|
||||
// or use safe atomic access as in `mi_page_set_has_interior_pointers`.
|
||||
// (we can access the page though since the just allocated pointer keeps it alive)
|
||||
mi_page_t* page = _mi_ptr_page(p);
|
||||
if (aligned_p != p) {
|
||||
mi_page_set_has_interior_pointers(page, true);
|
||||
#if MI_GUARDED
|
||||
// set tag to aligned so mi_usable_size works with guard pages
|
||||
if (adjust >= sizeof(mi_block_t)) {
|
||||
mi_block_t* const block = (mi_block_t*)p;
|
||||
block->next = MI_BLOCK_TAG_ALIGNED;
|
||||
}
|
||||
#endif
|
||||
_mi_padding_shrink(page, (mi_block_t*)p, adjust + size);
|
||||
}
|
||||
// todo: expand padding if overallocated ?
|
||||
|
||||
mi_assert_internal(mi_page_usable_block_size(page) >= adjust + size);
|
||||
mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0);
|
||||
mi_assert_internal(mi_usable_size(aligned_p)>=size);
|
||||
mi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust);
|
||||
#if MI_DEBUG > 1
|
||||
mi_page_t* const apage = _mi_ptr_page(aligned_p);
|
||||
void* unalign_p = _mi_page_ptr_unalign(apage, aligned_p);
|
||||
mi_assert_internal(p == unalign_p);
|
||||
#endif
|
||||
|
||||
// now zero the block if needed
|
||||
//if (alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) {
|
||||
// // for the tracker, on huge aligned allocations only from the start of the large block is defined
|
||||
// mi_track_mem_undefined(aligned_p, size);
|
||||
// if (zero) {
|
||||
// _mi_memzero_aligned(aligned_p, mi_usable_size(aligned_p));
|
||||
// }
|
||||
//}
|
||||
|
||||
if (p != aligned_p) {
|
||||
mi_track_align(p,aligned_p,adjust,mi_usable_size(aligned_p));
|
||||
#if MI_GUARDED
|
||||
mi_track_mem_defined(p, sizeof(mi_block_t));
|
||||
#endif
|
||||
}
|
||||
return aligned_p;
|
||||
}
|
||||
|
||||
// Generic primitive aligned allocation -- split out for better codegen
|
||||
static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept
|
||||
{
|
||||
mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment));
|
||||
// we don't allocate more than MI_MAX_ALLOC_SIZE (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
|
||||
if mi_unlikely(size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)) {
|
||||
_mi_error_message(EOVERFLOW, "aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// use regular allocation if it is guaranteed to fit the alignment constraints.
|
||||
// this is important to try as the fast path in `mi_theap_malloc_zero_aligned` only works when there exist
|
||||
// a page with the right block size, and if we always use the over-alloc fallback that would never happen.
|
||||
if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) {
|
||||
void* p = mi_theap_malloc_zero_no_guarded(theap, size, zero, usable);
|
||||
mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0);
|
||||
const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0;
|
||||
if mi_likely(is_aligned_or_null) {
|
||||
return p;
|
||||
}
|
||||
else {
|
||||
// this should never happen if the `mi_malloc_is_naturally_aligned` check is correct..
|
||||
mi_assert(false);
|
||||
mi_free(p);
|
||||
}
|
||||
}
|
||||
|
||||
// fall back to over-allocation
|
||||
return mi_theap_malloc_zero_aligned_at_overalloc(theap,size,alignment,offset,zero,usable);
|
||||
}
|
||||
|
||||
|
||||
// Primitive aligned allocation
|
||||
static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept
|
||||
{
|
||||
// note: we don't require `size > offset`, we just guarantee that the address at offset is aligned regardless of the allocated size.
|
||||
if mi_unlikely(alignment == 0 || !_mi_is_power_of_two(alignment)) { // require power-of-two (see <https://en.cppreference.com/w/c/memory/aligned_alloc>)
|
||||
#if MI_DEBUG > 0
|
||||
_mi_error_message(EOVERFLOW, "aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu)\n", size, alignment);
|
||||
#endif
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#if MI_GUARDED
|
||||
#if MI_THEAP_INITASNULL
|
||||
if mi_likely(theap!=NULL)
|
||||
#endif
|
||||
if (offset==0 && alignment < MI_PAGE_MAX_OVERALLOC_ALIGN && mi_theap_malloc_use_guarded(theap,size)) {
|
||||
return mi_theap_malloc_guarded_aligned(theap, size, alignment, zero);
|
||||
}
|
||||
#endif
|
||||
|
||||
// try first if there happens to be a small block available with just the right alignment
|
||||
// since most small power-of-2 blocks (under MI_PAGE_MAX_BLOCK_START_ALIGN2) are already
|
||||
// naturally aligned this can be often the case.
|
||||
#if MI_THEAP_INITASNULL
|
||||
if mi_likely(theap!=NULL)
|
||||
#endif
|
||||
{
|
||||
if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size) {
|
||||
const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)`
|
||||
const size_t padsize = size + MI_PADDING_SIZE;
|
||||
mi_page_t* page = _mi_theap_get_free_small_page(theap, padsize);
|
||||
if mi_likely(page->free != NULL) {
|
||||
const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0;
|
||||
if mi_likely(is_aligned)
|
||||
{
|
||||
if (usable!=NULL) { *usable = mi_page_usable_block_size(page); }
|
||||
void* p = _mi_page_malloc_zero(theap, page, padsize, zero);
|
||||
mi_assert_internal(p != NULL);
|
||||
mi_assert_internal(((uintptr_t)p + offset) % alignment == 0);
|
||||
mi_track_malloc(p, size, zero);
|
||||
return p;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// fallback to generic aligned allocation
|
||||
return mi_theap_malloc_zero_aligned_at_generic(theap, size, alignment, offset, zero, usable);
|
||||
}
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Internal mi_theap_malloc_aligned / mi_malloc_aligned
|
||||
// ------------------------------------------------------
|
||||
|
||||
static mi_decl_restrict void* mi_theap_malloc_aligned_at(mi_theap_t* theap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
|
||||
return mi_theap_malloc_zero_aligned_at(theap, size, alignment, offset, false, NULL);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept {
|
||||
return mi_theap_malloc_aligned_at(theap, size, alignment, 0);
|
||||
}
|
||||
|
||||
static mi_decl_restrict void* mi_theap_zalloc_aligned_at(mi_theap_t* theap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
|
||||
return mi_theap_malloc_zero_aligned_at(theap, size, alignment, offset, true, NULL);
|
||||
}
|
||||
|
||||
static mi_decl_restrict void* mi_theap_zalloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept {
|
||||
return mi_theap_zalloc_aligned_at(theap, size, alignment, 0);
|
||||
}
|
||||
|
||||
static mi_decl_restrict void* mi_theap_calloc_aligned_at(mi_theap_t* theap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
|
||||
size_t total;
|
||||
if (mi_count_size_overflow(count, size, &total)) return NULL;
|
||||
return mi_theap_zalloc_aligned_at(theap, total, alignment, offset);
|
||||
}
|
||||
|
||||
static mi_decl_restrict void* mi_theap_calloc_aligned(mi_theap_t* theap, size_t count, size_t size, size_t alignment) mi_attr_noexcept {
|
||||
return mi_theap_calloc_aligned_at(theap, count, size, alignment, 0);
|
||||
}
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Aligned Allocation
|
||||
// ------------------------------------------------------
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
|
||||
return mi_theap_malloc_aligned_at(_mi_theap_default(), size, alignment, offset);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept {
|
||||
return mi_theap_malloc_aligned(_mi_theap_default(), size, alignment);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept {
|
||||
return mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, false, block_size);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
|
||||
return mi_theap_zalloc_aligned_at(_mi_theap_default(), size, alignment, offset);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept {
|
||||
return mi_theap_zalloc_aligned(_mi_theap_default(), size, alignment);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept {
|
||||
return mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, true, block_size);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
|
||||
return mi_theap_calloc_aligned_at(_mi_theap_default(), count, size, alignment, offset);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept {
|
||||
return mi_theap_calloc_aligned(_mi_theap_default(), count, size, alignment);
|
||||
}
|
||||
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
|
||||
return mi_theap_malloc_aligned_at(_mi_heap_theap(heap), size, alignment, offset);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept {
|
||||
return mi_theap_malloc_aligned(_mi_heap_theap(heap), size, alignment);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
|
||||
return mi_theap_zalloc_aligned_at(_mi_heap_theap(heap), size, alignment, offset);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept {
|
||||
return mi_theap_zalloc_aligned(_mi_heap_theap(heap), size, alignment);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
|
||||
return mi_theap_calloc_aligned_at(_mi_heap_theap(heap), count, size, alignment, offset);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept {
|
||||
return mi_theap_calloc_aligned(_mi_heap_theap(heap), count, size, alignment);
|
||||
}
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Aligned re-allocation
|
||||
// ------------------------------------------------------
|
||||
|
||||
static void* mi_theap_realloc_zero_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero) mi_attr_noexcept {
|
||||
mi_assert(alignment > 0);
|
||||
if (alignment <= sizeof(uintptr_t) && offset==0) return _mi_theap_realloc_zero(theap,p,newsize,zero,NULL,NULL);
|
||||
if (p == NULL) return mi_theap_malloc_zero_aligned_at(theap,newsize,alignment,offset,zero,NULL);
|
||||
size_t size = mi_usable_size(p);
|
||||
if (newsize <= size && newsize >= (size - (size / 2))
|
||||
&& (((uintptr_t)p + offset) % alignment) == 0) {
|
||||
return p; // reallocation still fits, is aligned and not more than 25% waste
|
||||
}
|
||||
else {
|
||||
// note: we don't zero allocate upfront so we only zero initialize the expanded part
|
||||
void* newp = mi_theap_malloc_aligned_at(theap,newsize,alignment,offset);
|
||||
if (newp != NULL) {
|
||||
if (zero && newsize > size) {
|
||||
// also set last word in the previous allocation to zero to ensure any padding is zero-initialized
|
||||
size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);
|
||||
_mi_memzero((uint8_t*)newp + start, newsize - start);
|
||||
}
|
||||
_mi_memcpy_aligned(newp, p, (newsize > size ? size : newsize));
|
||||
mi_free(p); // only free if successful
|
||||
}
|
||||
return newp;
|
||||
}
|
||||
}
|
||||
|
||||
static void* mi_theap_realloc_zero_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, bool zero) mi_attr_noexcept {
|
||||
mi_assert(alignment > 0);
|
||||
if (alignment <= sizeof(uintptr_t)) return _mi_theap_realloc_zero(theap,p,newsize,zero,NULL,NULL);
|
||||
return mi_theap_realloc_zero_aligned_at(theap,p,newsize,alignment,0,zero);
|
||||
}
|
||||
|
||||
static void* mi_theap_realloc_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
|
||||
return mi_theap_realloc_zero_aligned_at(theap,p,newsize,alignment,offset,false);
|
||||
}
|
||||
|
||||
static void* mi_theap_realloc_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
|
||||
return mi_theap_realloc_zero_aligned(theap,p,newsize,alignment,false);
|
||||
}
|
||||
|
||||
static void* mi_theap_rezalloc_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
|
||||
return mi_theap_realloc_zero_aligned_at(theap, p, newsize, alignment, offset, true);
|
||||
}
|
||||
|
||||
static void* mi_theap_rezalloc_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
|
||||
return mi_theap_realloc_zero_aligned(theap, p, newsize, alignment, true);
|
||||
}
|
||||
|
||||
static void* mi_theap_recalloc_aligned_at(mi_theap_t* theap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
|
||||
size_t total;
|
||||
if (mi_count_size_overflow(newcount, size, &total)) return NULL;
|
||||
return mi_theap_rezalloc_aligned_at(theap, p, total, alignment, offset);
|
||||
}
|
||||
|
||||
static void* mi_theap_recalloc_aligned(mi_theap_t* theap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept {
|
||||
size_t total;
|
||||
if (mi_count_size_overflow(newcount, size, &total)) return NULL;
|
||||
return mi_theap_rezalloc_aligned(theap, p, total, alignment);
|
||||
}
|
||||
|
||||
|
||||
mi_decl_nodiscard void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
|
||||
return mi_theap_realloc_aligned_at(_mi_theap_default(), p, newsize, alignment, offset);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
|
||||
return mi_theap_realloc_aligned(_mi_theap_default(), p, newsize, alignment);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
|
||||
return mi_theap_rezalloc_aligned_at(_mi_theap_default(), p, newsize, alignment, offset);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
|
||||
return mi_theap_rezalloc_aligned(_mi_theap_default(), p, newsize, alignment);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
|
||||
return mi_theap_recalloc_aligned_at(_mi_theap_default(), p, newcount, size, alignment, offset);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept {
|
||||
return mi_theap_recalloc_aligned(_mi_theap_default(), p, newcount, size, alignment);
|
||||
}
|
||||
|
||||
|
||||
mi_decl_nodiscard void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
|
||||
return mi_theap_realloc_aligned_at(_mi_heap_theap(heap), p, newsize, alignment, offset);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
|
||||
return mi_theap_realloc_aligned(_mi_heap_theap(heap), p, newsize, alignment);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
|
||||
return mi_theap_rezalloc_aligned_at(_mi_heap_theap(heap), p, newsize, alignment, offset);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
|
||||
return mi_theap_rezalloc_aligned(_mi_heap_theap(heap), p, newsize, alignment);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
|
||||
return mi_theap_recalloc_aligned_at(_mi_heap_theap(heap), p, newcount, size, alignment, offset);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept {
|
||||
return mi_theap_recalloc_aligned(_mi_heap_theap(heap), p, newcount, size, alignment);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,396 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
#if !defined(MI_IN_ALLOC_C)
|
||||
#error "this file should be included from 'alloc.c' (so aliases can work)"
|
||||
#endif
|
||||
|
||||
|
||||
#if defined(MI_MALLOC_OVERRIDE) && !defined(_DLL)
|
||||
|
||||
#if defined(__APPLE__)
|
||||
#include <AvailabilityMacros.h>
|
||||
mi_decl_externc void vfree(void* p);
|
||||
mi_decl_externc size_t malloc_size(const void* p);
|
||||
mi_decl_externc size_t malloc_good_size(size_t size);
|
||||
#endif
|
||||
|
||||
// helper definition for C override of C++ new
|
||||
typedef void* mi_nothrow_t;
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Override system malloc
|
||||
// ------------------------------------------------------
|
||||
|
||||
#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) && !MI_TRACK_ENABLED
|
||||
// gcc, clang: use aliasing to alias the exported function to one of our `mi_` functions
|
||||
#if (defined(__GNUC__) && __GNUC__ >= 9)
|
||||
#pragma GCC diagnostic ignored "-Wattributes" // or we get warnings that nodiscard is ignored on a forward
|
||||
#define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"), copy(fun)));
|
||||
#else
|
||||
#define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default")));
|
||||
#endif
|
||||
#define MI_FORWARD1(fun,x) MI_FORWARD(fun)
|
||||
#define MI_FORWARD2(fun,x,y) MI_FORWARD(fun)
|
||||
#define MI_FORWARD3(fun,x,y,z) MI_FORWARD(fun)
|
||||
#define MI_FORWARD0(fun,x) MI_FORWARD(fun)
|
||||
#define MI_FORWARD02(fun,x,y) MI_FORWARD(fun)
|
||||
#else
|
||||
// otherwise use forwarding by calling our `mi_` function
|
||||
#define MI_FORWARD1(fun,x) { return fun(x); }
|
||||
#define MI_FORWARD2(fun,x,y) { return fun(x,y); }
|
||||
#define MI_FORWARD3(fun,x,y,z) { return fun(x,y,z); }
|
||||
#define MI_FORWARD0(fun,x) { fun(x); }
|
||||
#define MI_FORWARD02(fun,x,y) { fun(x,y); }
|
||||
#endif
|
||||
|
||||
#if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE)
|
||||
// define MI_OSX_IS_INTERPOSED as we should not provide forwarding definitions for
|
||||
// functions that are interposed (or the interposing does not work)
|
||||
#define MI_OSX_IS_INTERPOSED
|
||||
|
||||
mi_decl_externc size_t mi_malloc_size_checked(void *p) {
|
||||
if (!mi_is_in_heap_region(p)) return 0;
|
||||
return mi_usable_size(p);
|
||||
}
|
||||
|
||||
// use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1`
|
||||
// See: <https://books.google.com/books?id=K8vUkpOXhN4C&pg=PA73>
|
||||
struct mi_interpose_s {
|
||||
const void* replacement;
|
||||
const void* target;
|
||||
};
|
||||
#define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun }
|
||||
#define MI_INTERPOSE_MI(fun) MI_INTERPOSE_FUN(fun,mi_##fun)
|
||||
|
||||
#define MI_INTERPOSE_DECLS(name) __attribute__((used)) static struct mi_interpose_s name[] __attribute__((section("__DATA, __interpose")))
|
||||
|
||||
MI_INTERPOSE_DECLS(_mi_interposes) =
|
||||
{
|
||||
MI_INTERPOSE_MI(malloc),
|
||||
MI_INTERPOSE_MI(calloc),
|
||||
MI_INTERPOSE_MI(realloc),
|
||||
MI_INTERPOSE_MI(strdup),
|
||||
MI_INTERPOSE_MI(realpath),
|
||||
MI_INTERPOSE_MI(posix_memalign),
|
||||
MI_INTERPOSE_MI(reallocf),
|
||||
MI_INTERPOSE_MI(valloc),
|
||||
MI_INTERPOSE_FUN(malloc_size,mi_malloc_size_checked),
|
||||
MI_INTERPOSE_MI(malloc_good_size),
|
||||
#ifdef MI_OSX_ZONE
|
||||
// we interpose malloc_default_zone in alloc-override-osx.c so we can use mi_free safely
|
||||
MI_INTERPOSE_MI(free),
|
||||
MI_INTERPOSE_FUN(vfree,mi_free),
|
||||
#else
|
||||
// sometimes code allocates from default zone but deallocates using plain free :-( (like NxHashResizeToCapacity <https://github.com/nneonneo/osx-10.9-opensource/blob/master/objc4-551.1/runtime/hashtable2.mm>)
|
||||
MI_INTERPOSE_FUN(free,mi_cfree), // use safe free that checks if pointers are from us
|
||||
MI_INTERPOSE_FUN(vfree,mi_cfree),
|
||||
#endif
|
||||
};
|
||||
#if defined(MAC_OS_X_VERSION_10_7) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7)
|
||||
MI_INTERPOSE_DECLS(_mi_interposes_10_7) = { MI_INTERPOSE_MI(strndup) };
|
||||
#endif
|
||||
#if defined(MAC_OS_X_VERSION_10_15) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_15)
|
||||
MI_INTERPOSE_DECLS(_mi_interposes_10_15) = { MI_INTERPOSE_MI(aligned_alloc) };
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
void _ZdlPv(void* p); // delete
|
||||
void _ZdaPv(void* p); // delete[]
|
||||
void _ZdlPvm(void* p, size_t n); // delete
|
||||
void _ZdaPvm(void* p, size_t n); // delete[]
|
||||
void* _Znwm(size_t n); // new
|
||||
void* _Znam(size_t n); // new[]
|
||||
void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new nothrow
|
||||
void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new[] nothrow
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
__attribute__((used)) static struct mi_interpose_s _mi_cxx_interposes[] __attribute__((section("__DATA, __interpose"))) =
|
||||
{
|
||||
MI_INTERPOSE_FUN(_ZdlPv,mi_free),
|
||||
MI_INTERPOSE_FUN(_ZdaPv,mi_free),
|
||||
MI_INTERPOSE_FUN(_ZdlPvm,mi_free_size),
|
||||
MI_INTERPOSE_FUN(_ZdaPvm,mi_free_size),
|
||||
MI_INTERPOSE_FUN(_Znwm,mi_new),
|
||||
MI_INTERPOSE_FUN(_Znam,mi_new),
|
||||
MI_INTERPOSE_FUN(_ZnwmRKSt9nothrow_t,mi_new_nothrow),
|
||||
MI_INTERPOSE_FUN(_ZnamRKSt9nothrow_t,mi_new_nothrow),
|
||||
};
|
||||
|
||||
#elif defined(_MSC_VER)
|
||||
_Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRT_HYBRIDPATCHABLE
|
||||
void* __cdecl _expand(_Pre_notnull_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size) {
|
||||
return mi_expand(_Block, _Size);
|
||||
}
|
||||
_Check_return_ _ACRTIMP
|
||||
size_t __cdecl _msize_base(_Pre_notnull_ void* _Block) _CRT_NOEXCEPT {
|
||||
return mi_malloc_size(_Block);
|
||||
}
|
||||
_Check_return_ _ACRTIMP _CRT_HYBRIDPATCHABLE
|
||||
size_t __cdecl _msize(_Pre_notnull_ void* _Block) {
|
||||
return mi_malloc_size(_Block);
|
||||
}
|
||||
_ACRTIMP
|
||||
void __cdecl _free_base(_Pre_maybenull_ _Post_invalid_ void* _Block) {
|
||||
mi_free(_Block);
|
||||
}
|
||||
_ACRTIMP _CRT_HYBRIDPATCHABLE
|
||||
void __cdecl free(_Pre_maybenull_ _Post_invalid_ void* _Block) {
|
||||
mi_free(_Block);
|
||||
}
|
||||
_Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRT_JIT_INTRINSIC _CRTRESTRICT _CRT_HYBRIDPATCHABLE
|
||||
void* __cdecl malloc(_In_ _CRT_GUARDOVERFLOW size_t _Size) {
|
||||
return mi_malloc(_Size);
|
||||
}
|
||||
_Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
|
||||
void* __cdecl _malloc_base(_In_ size_t _Size) {
|
||||
return mi_malloc(_Size);
|
||||
}
|
||||
_Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
|
||||
void* __cdecl _realloc_base(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ size_t _Size) {
|
||||
return mi_realloc(_Block, _Size);
|
||||
}
|
||||
_Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT _CRT_HYBRIDPATCHABLE
|
||||
void* __cdecl realloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size) {
|
||||
return mi_realloc(_Block, _Size);
|
||||
}
|
||||
_Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
|
||||
void* __cdecl _calloc_base(_In_ size_t _Count, _In_ size_t _Size) {
|
||||
return mi_calloc(_Count, _Size);
|
||||
}
|
||||
_Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRT_JIT_INTRINSIC _CRTALLOCATOR _CRTRESTRICT
|
||||
void* __cdecl calloc(_In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size) {
|
||||
return mi_calloc(_Count, _Size);
|
||||
}
|
||||
_Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
|
||||
void* __cdecl _recalloc_base(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ size_t _Count, _In_ size_t _Size) {
|
||||
return mi_recalloc(_Block, _Count, _Size);
|
||||
}
|
||||
_Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
|
||||
void* __cdecl _recalloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size) {
|
||||
return mi_recalloc(_Block, _Count, _Size);
|
||||
}
|
||||
_ACRTIMP
|
||||
void __cdecl _aligned_free(_Pre_maybenull_ _Post_invalid_ void* _Block) {
|
||||
mi_free(_Block);
|
||||
}
|
||||
_Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
|
||||
void* __cdecl _aligned_malloc(_In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment) {
|
||||
return mi_malloc_aligned(_Size, _Alignment);
|
||||
}
|
||||
_Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
|
||||
void* __cdecl _aligned_offset_malloc(_In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment, _In_ size_t _Offset) {
|
||||
return mi_malloc_aligned_at(_Size, _Alignment, _Offset);
|
||||
}
|
||||
_Check_return_ _ACRTIMP
|
||||
size_t __cdecl _aligned_msize(_Pre_notnull_ void* _Block, _In_ size_t _Alignment, _In_ size_t _Offset) {
|
||||
MI_UNUSED(_Alignment); MI_UNUSED(_Offset); return mi_malloc_size(_Block);
|
||||
}
|
||||
_Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
|
||||
void* __cdecl _aligned_offset_realloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment, _In_ size_t _Offset) {
|
||||
return mi_realloc_aligned_at(_Block, _Size, _Alignment, _Offset);
|
||||
}
|
||||
_Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
|
||||
void* __cdecl _aligned_offset_recalloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment, _In_ size_t _Offset) {
|
||||
return mi_recalloc_aligned_at(_Block, _Count, _Size, _Alignment, _Offset);
|
||||
}
|
||||
_Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
|
||||
void* __cdecl _aligned_realloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment) {
|
||||
return mi_realloc_aligned(_Block, _Size, _Alignment);
|
||||
}
|
||||
_Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
|
||||
void* __cdecl _aligned_recalloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment) {
|
||||
return mi_recalloc_aligned(_Block, _Count, _Size, _Alignment);
|
||||
}
|
||||
#else
|
||||
// On all other systems forward allocation primitives to our API
|
||||
mi_decl_export void* malloc(size_t size) MI_FORWARD1(mi_malloc, size)
|
||||
mi_decl_export void* calloc(size_t size, size_t n) MI_FORWARD2(mi_calloc, size, n)
|
||||
mi_decl_export void* realloc(void* p, size_t newsize) MI_FORWARD2(mi_realloc, p, newsize)
|
||||
mi_decl_export void free(void* p) MI_FORWARD0(mi_free, p)
|
||||
// In principle we do not need to forward `strdup`/`strndup` but on some systems these do not use `malloc` internally (but a more primitive call)
|
||||
// We only override if `strdup` is not a macro (as on some older libc's, see issue #885)
|
||||
#if !defined(strdup)
|
||||
mi_decl_export char* strdup(const char* str) MI_FORWARD1(mi_strdup, str)
|
||||
#endif
|
||||
#if !defined(strndup) && (!defined(__APPLE__) || (defined(MAC_OS_X_VERSION_10_7) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7))
|
||||
mi_decl_export char* strndup(const char* str, size_t n) MI_FORWARD2(mi_strndup, str, n)
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__)
|
||||
#pragma GCC visibility push(default)
|
||||
#endif
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Override new/delete
|
||||
// This is not really necessary as they usually call
|
||||
// malloc/free anyway, but it improves performance.
|
||||
// ------------------------------------------------------
|
||||
#ifdef __cplusplus
|
||||
// ------------------------------------------------------
|
||||
// With a C++ compiler we override the new/delete operators.
|
||||
// see <https://en.cppreference.com/w/cpp/memory/new/operator_new>
|
||||
// ------------------------------------------------------
|
||||
#include <new>
|
||||
|
||||
#ifndef MI_OSX_IS_INTERPOSED
|
||||
void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p)
|
||||
void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p)
|
||||
|
||||
void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n)
|
||||
void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n)
|
||||
|
||||
void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { MI_UNUSED(tag); return mi_new_nothrow(n); }
|
||||
void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { MI_UNUSED(tag); return mi_new_nothrow(n); }
|
||||
|
||||
#if (__cplusplus >= 201402L || _MSC_VER >= 1916)
|
||||
void operator delete (void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n)
|
||||
void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n)
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if (__cplusplus > 201402L && defined(__cpp_aligned_new)) && (!defined(__GNUC__) || (__GNUC__ > 5))
|
||||
void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
|
||||
void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
|
||||
void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
|
||||
void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
|
||||
void operator delete (void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
|
||||
void operator delete[](void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
|
||||
|
||||
void* operator new( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
|
||||
void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
|
||||
void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast<size_t>(al)); }
|
||||
void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast<size_t>(al)); }
|
||||
#endif
|
||||
|
||||
#elif (defined(__GNUC__) || defined(__clang__))
|
||||
// ------------------------------------------------------
|
||||
// Override by defining the mangled C++ names of the operators (as
|
||||
// used by GCC and CLang).
|
||||
// See <https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling>
|
||||
// ------------------------------------------------------
|
||||
|
||||
void _ZdlPv(void* p) MI_FORWARD0(mi_free,p) // delete
|
||||
void _ZdaPv(void* p) MI_FORWARD0(mi_free,p) // delete[]
|
||||
void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n)
|
||||
void _ZdaPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n)
|
||||
|
||||
void _ZdlPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); }
|
||||
void _ZdaPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); }
|
||||
void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); }
|
||||
void _ZdaPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); }
|
||||
|
||||
void _ZdlPvRKSt9nothrow_t(void* p, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free(p); } // operator delete(void*, std::nothrow_t const&)
|
||||
void _ZdaPvRKSt9nothrow_t(void* p, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free(p); } // operator delete[](void*, std::nothrow_t const&)
|
||||
void _ZdlPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete(void*, std::align_val_t, std::nothrow_t const&)
|
||||
void _ZdaPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete[](void*, std::align_val_t, std::nothrow_t const&)
|
||||
|
||||
#if (MI_INTPTR_SIZE==8) || (MI_INTPTR_SIZE==4 && defined(__EMSCRIPTEN__)) // pr #1257
|
||||
void* _Znwm(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit
|
||||
void* _Znam(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit
|
||||
void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }
|
||||
void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }
|
||||
void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al)
|
||||
void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al)
|
||||
void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
|
||||
void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
|
||||
#elif (MI_INTPTR_SIZE==4)
|
||||
void* _Znwj(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit
|
||||
void* _Znaj(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit
|
||||
void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }
|
||||
void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }
|
||||
void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al)
|
||||
void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al)
|
||||
void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
|
||||
void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
|
||||
#else
|
||||
#error "define overloads for new/delete for this platform (just for performance, can be skipped)"
|
||||
#endif
|
||||
#endif // __cplusplus
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Further Posix & Unix functions definitions
|
||||
// ------------------------------------------------------
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#ifndef MI_OSX_IS_INTERPOSED
|
||||
// Forward Posix/Unix calls as well
|
||||
void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize)
|
||||
size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p)
|
||||
#if !defined(__ANDROID__) && !defined(__FreeBSD__) && !defined(__DragonFly__)
|
||||
size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p)
|
||||
#else
|
||||
size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p)
|
||||
#endif
|
||||
|
||||
// No forwarding here due to aliasing/name mangling issues
|
||||
void* valloc(size_t size) { return mi_valloc(size); }
|
||||
void vfree(void* p) { mi_free(p); }
|
||||
size_t malloc_good_size(size_t size) { return mi_malloc_good_size(size); }
|
||||
int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); }
|
||||
|
||||
// `aligned_alloc` is only available when __USE_ISOC11 is defined.
|
||||
// Note: it seems __USE_ISOC11 is not defined in musl (and perhaps other libc's) so we only check
|
||||
// for it if using glibc.
|
||||
// Note: Conda has a custom glibc where `aligned_alloc` is declared `static inline` and we cannot
|
||||
// override it, but both _ISOC11_SOURCE and __USE_ISOC11 are undefined in Conda GCC7 or GCC9.
|
||||
// Fortunately, in the case where `aligned_alloc` is declared as `static inline` it
|
||||
// uses internally `memalign`, `posix_memalign`, or `_aligned_malloc` so we can avoid overriding it ourselves.
|
||||
#if !defined(__GLIBC__) || __USE_ISOC11
|
||||
void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); }
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// no forwarding here due to aliasing/name mangling issues
|
||||
void cfree(void* p) { mi_free(p); }
|
||||
void* pvalloc(size_t size) { return mi_pvalloc(size); }
|
||||
void* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); }
|
||||
#if !defined(_WIN32)
|
||||
void* _aligned_malloc(size_t size, size_t alignment) { return mi_malloc_aligned(size,alignment); }
|
||||
#endif
|
||||
void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); }
|
||||
// some systems define reallocarr so mark it as a weak symbol (#751)
|
||||
mi_decl_weak int reallocarr(void* p, size_t count, size_t size) { return mi_reallocarr(p, count, size); }
|
||||
|
||||
#if defined(__wasi__)
|
||||
// forward __libc interface (see PR #667)
|
||||
void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc, size)
|
||||
void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc, count, size)
|
||||
void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc, p, size)
|
||||
void __libc_free(void* p) MI_FORWARD0(mi_free, p)
|
||||
void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); }
|
||||
|
||||
#elif defined(__linux__)
|
||||
// forward __libc interface (needed for glibc-based and musl-based Linux distributions)
|
||||
void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size)
|
||||
void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc,count,size)
|
||||
void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc,p,size)
|
||||
void __libc_free(void* p) MI_FORWARD0(mi_free,p)
|
||||
void __libc_cfree(void* p) MI_FORWARD0(mi_free,p)
|
||||
|
||||
void* __libc_valloc(size_t size) { return mi_valloc(size); }
|
||||
void* __libc_pvalloc(size_t size) { return mi_pvalloc(size); }
|
||||
void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment,size); }
|
||||
int __posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p,alignment,size); }
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__)
|
||||
#pragma GCC visibility pop
|
||||
#endif
|
||||
|
||||
#endif // MI_MALLOC_OVERRIDE
|
||||
@@ -0,0 +1,202 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
// ------------------------------------------------------------------------
|
||||
// mi prefixed publi definitions of various Posix, Unix, and C++ functions
|
||||
// for convenience and used when overriding these functions.
|
||||
// ------------------------------------------------------------------------
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc/internal.h"
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Posix & Unix functions definitions
|
||||
// ------------------------------------------------------
|
||||
|
||||
#include <errno.h>
|
||||
#include <string.h> // memset
|
||||
#include <stdlib.h> // getenv
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#pragma warning(disable:4996) // getenv _wgetenv
|
||||
#endif
|
||||
|
||||
#ifndef EINVAL
|
||||
#define EINVAL 22
|
||||
#endif
|
||||
#ifndef ENOMEM
|
||||
#define ENOMEM 12
|
||||
#endif
|
||||
|
||||
|
||||
mi_decl_nodiscard size_t mi_malloc_size(const void* p) mi_attr_noexcept {
|
||||
// if (!mi_is_in_heap_region(p)) return 0;
|
||||
return mi_usable_size(p);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept {
|
||||
// if (!mi_is_in_heap_region(p)) return 0;
|
||||
return mi_usable_size(p);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard size_t mi_malloc_good_size(size_t size) mi_attr_noexcept {
|
||||
return mi_good_size(size);
|
||||
}
|
||||
|
||||
void mi_cfree(void* p) mi_attr_noexcept {
|
||||
if (mi_is_in_heap_region(p)) {
|
||||
mi_free(p);
|
||||
}
|
||||
}
|
||||
|
||||
int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept {
|
||||
// Note: The spec dictates we should not modify `*p` on an error. (issue#27)
|
||||
// <http://man7.org/linux/man-pages/man3/posix_memalign.3.html>
|
||||
if (p == NULL) return EINVAL;
|
||||
if ((alignment % sizeof(void*)) != 0) return EINVAL; // natural alignment
|
||||
// it is also required that alignment is a power of 2 and > 0; this is checked in `mi_malloc_aligned`
|
||||
if (alignment==0 || !_mi_is_power_of_two(alignment)) return EINVAL; // not a power of 2
|
||||
void* q = mi_malloc_aligned(size, alignment);
|
||||
if (q==NULL && size != 0) return ENOMEM;
|
||||
mi_assert_internal(_mi_is_aligned(q,alignment));
|
||||
*p = q;
|
||||
return 0;
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept {
|
||||
void* p = mi_malloc_aligned(size, alignment);
|
||||
mi_assert_internal(_mi_is_aligned(p,alignment));
|
||||
return p;
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept {
|
||||
return mi_memalign( _mi_os_page_size(), size );
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept {
|
||||
size_t psize = _mi_os_page_size();
|
||||
if (size >= SIZE_MAX - psize) return NULL; // overflow
|
||||
size_t asize = _mi_align_up(size, psize);
|
||||
return mi_malloc_aligned(asize, psize);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept {
|
||||
// C11 requires the size to be an integral multiple of the alignment, see <https://en.cppreference.com/w/c/memory/aligned_alloc>.
|
||||
// unfortunately, it turns out quite some programs pass a size that is not an integral multiple so skip this check..
|
||||
/* if mi_unlikely((size & (alignment - 1)) != 0) { // C11 requires alignment>0 && integral multiple, see <https://en.cppreference.com/w/c/memory/aligned_alloc>
|
||||
#if MI_DEBUG > 0
|
||||
_mi_error_message(EOVERFLOW, "(mi_)aligned_alloc requires the size to be an integral multiple of the alignment (size %zu, alignment %zu)\n", size, alignment);
|
||||
#endif
|
||||
return NULL;
|
||||
}
|
||||
*/
|
||||
// C11 also requires alignment to be a power-of-two (and > 0) which is checked in mi_malloc_aligned
|
||||
void* p = mi_malloc_aligned(size, alignment);
|
||||
mi_assert_internal(_mi_is_aligned(p,alignment));
|
||||
return p;
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept { // BSD <https://man.freebsd.org/cgi/man.cgi?query=reallocarray>
|
||||
size_t total;
|
||||
if mi_unlikely(mi_count_size_overflow(count, size, &total)) {
|
||||
errno = EOVERFLOW;
|
||||
return NULL;
|
||||
}
|
||||
void* newp = mi_realloc(p,total);
|
||||
if (newp==NULL) { errno = ENOMEM; }
|
||||
return newp;
|
||||
}
|
||||
|
||||
mi_decl_nodiscard int mi_reallocarr( void* ptrp, size_t count, size_t size ) mi_attr_noexcept { // NetBSD <https://man.netbsd.org/reallocarr.3>
|
||||
mi_assert(size != 0);
|
||||
mi_assert(ptrp != NULL);
|
||||
if (ptrp == NULL || size == 0) {
|
||||
return (errno = EINVAL);
|
||||
}
|
||||
size_t total;
|
||||
if mi_unlikely(mi_count_size_overflow(count, size, &total)) {
|
||||
return (errno = EOVERFLOW);
|
||||
}
|
||||
void** op = (void**)ptrp;
|
||||
if (total == 0) {
|
||||
free(*op);
|
||||
*op = NULL;
|
||||
return 0;
|
||||
}
|
||||
else {
|
||||
void* newp = mi_realloc(*op,total);
|
||||
if (newp == NULL) { return (errno = ENOMEM); }
|
||||
*op = newp;
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
void* mi__expand(void* p, size_t newsize) mi_attr_noexcept { // Microsoft
|
||||
void* res = mi_expand(p, newsize);
|
||||
if (res == NULL) { errno = ENOMEM; }
|
||||
return res;
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s) mi_attr_noexcept {
|
||||
if (s==NULL) return NULL;
|
||||
size_t wlen;
|
||||
for(wlen = 0; s[wlen] != 0 && wlen < PTRDIFF_MAX; wlen++) { } // prevent overflow on wlen+1
|
||||
size_t size;
|
||||
if (mi_mul_overflow(wlen+1, sizeof(wchar_t), &size) || size > PTRDIFF_MAX) return NULL;
|
||||
wchar_t* p = (wchar_t*)mi_malloc(size);
|
||||
if (p != NULL) {
|
||||
_mi_memcpy(p,s,size);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept {
|
||||
return (unsigned char*)mi_strdup((const char*)s);
|
||||
}
|
||||
|
||||
int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept {
|
||||
if (size != NULL) *size = 0;
|
||||
if (buf==NULL || name==NULL) return EINVAL;
|
||||
char* p = getenv(name);
|
||||
if (p==NULL) {
|
||||
*buf = NULL;
|
||||
}
|
||||
else {
|
||||
*buf = mi_strdup(p);
|
||||
if (*buf==NULL) return ENOMEM;
|
||||
if (size != NULL) { *size = _mi_strlen(p) + 1; } // cannot overflow as mi_strdup is limited to PTRDIFF_MAX
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name) mi_attr_noexcept {
|
||||
if (size != NULL) *size = 0;
|
||||
if (buf==NULL || name==NULL) return EINVAL;
|
||||
#if !defined(_WIN32) || (defined(WINAPI_FAMILY) && (WINAPI_FAMILY != WINAPI_FAMILY_DESKTOP_APP))
|
||||
// not supported
|
||||
*buf = NULL;
|
||||
return EINVAL;
|
||||
#else
|
||||
wchar_t* p = (wchar_t*)_wgetenv(name);
|
||||
if (p==NULL) {
|
||||
*buf = NULL;
|
||||
}
|
||||
else {
|
||||
*buf = mi_wcsdup(p);
|
||||
if (*buf==NULL) return ENOMEM;
|
||||
if (size != NULL) { *size = wcslen(p) + 1; } // cannot overflow as wcsdup is limited to PTRDIFF_MAX
|
||||
}
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { // Microsoft
|
||||
return mi_recalloc_aligned_at(p, newcount, size, alignment, offset);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { // Microsoft
|
||||
return mi_recalloc_aligned(p, newcount, size, alignment);
|
||||
}
|
||||
@@ -0,0 +1,885 @@
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#ifndef _DEFAULT_SOURCE
|
||||
#define _DEFAULT_SOURCE // for realpath() on Linux
|
||||
#endif
|
||||
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc/internal.h"
|
||||
#include "mimalloc/atomic.h"
|
||||
#include "mimalloc/prim.h" // _mi_prim_thread_id()
|
||||
|
||||
#include <string.h> // memset, strlen (for mi_strdup)
|
||||
#include <stdlib.h> // malloc, abort
|
||||
|
||||
#define MI_IN_ALLOC_C
|
||||
#include "alloc-override.c"
|
||||
#include "free.c"
|
||||
#undef MI_IN_ALLOC_C
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Allocation
|
||||
// ------------------------------------------------------
|
||||
|
||||
// Fast allocation in a page: just pop from the free list.
|
||||
// Fall back to generic allocation only if the list is empty.
|
||||
// Note: in release mode the (inlined) routine is about 7 instructions with a single test.
|
||||
static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero, size_t* usable) mi_attr_noexcept
|
||||
{
|
||||
if (page->block_size != 0) { // not the empty theap
|
||||
mi_assert_internal(mi_page_block_size(page) >= size);
|
||||
mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
|
||||
mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
|
||||
}
|
||||
|
||||
// check the free list
|
||||
mi_block_t* const block = page->free;
|
||||
if mi_unlikely(block == NULL) {
|
||||
return _mi_malloc_generic(theap, size, (zero ? 1 : 0), usable);
|
||||
}
|
||||
mi_assert_internal(block != NULL && _mi_ptr_page(block) == page);
|
||||
if (usable != NULL) { *usable = mi_page_usable_block_size(page); };
|
||||
|
||||
// pop from the free list
|
||||
page->free = mi_block_next(page, block);
|
||||
page->used++;
|
||||
mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page);
|
||||
mi_assert_internal(page->block_size < MI_MAX_ALIGN_SIZE || _mi_is_aligned(block, MI_MAX_ALIGN_SIZE));
|
||||
|
||||
#if MI_DEBUG>3
|
||||
if (page->free_is_zero && size > sizeof(*block)) {
|
||||
mi_assert_expensive(mi_mem_is_zero(block+1,size - sizeof(*block)));
|
||||
}
|
||||
#endif
|
||||
|
||||
// allow use of the block internally
|
||||
// note: when tracking we need to avoid ever touching the MI_PADDING since
|
||||
// that is tracked by valgrind etc. as non-accessible (through the red-zone, see `mimalloc/track.h`)
|
||||
const size_t bsize = mi_page_usable_block_size(page);
|
||||
mi_track_mem_undefined(block, bsize);
|
||||
|
||||
#if (MI_STAT>0)
|
||||
if (bsize <= MI_LARGE_MAX_OBJ_SIZE) {
|
||||
mi_theap_stat_increase(theap, malloc_normal, bsize);
|
||||
#if (MI_STAT>1)
|
||||
mi_theap_stat_counter_increase(theap, malloc_normal_count, 1);
|
||||
const size_t bin = _mi_bin(bsize);
|
||||
mi_theap_stat_increase(theap, malloc_bins[bin], 1);
|
||||
mi_theap_stat_increase(theap, malloc_requested, size - MI_PADDING_SIZE);
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
// zero the block? note: we need to zero the full block size (issue #63)
|
||||
if mi_likely(!zero) {
|
||||
// #if MI_SECURE
|
||||
block->next = 0; // don't leak internal data
|
||||
// #endif
|
||||
#if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN
|
||||
if (!mi_page_is_huge(page)) { memset(block, MI_DEBUG_UNINIT, bsize); }
|
||||
#endif
|
||||
}
|
||||
else {
|
||||
if (!page->free_is_zero) {
|
||||
_mi_memzero_aligned(block, bsize);
|
||||
}
|
||||
else {
|
||||
block->next = 0;
|
||||
mi_track_mem_defined(block, bsize);
|
||||
}
|
||||
}
|
||||
|
||||
#if MI_PADDING // && !MI_TRACK_ENABLED
|
||||
mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + bsize);
|
||||
ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE));
|
||||
#if (MI_DEBUG>=2)
|
||||
mi_assert_internal(delta >= 0 && bsize >= (size - MI_PADDING_SIZE + delta));
|
||||
#endif
|
||||
mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess
|
||||
padding->canary = mi_ptr_encode_canary(page,block,page->keys);
|
||||
padding->delta = (uint32_t)(delta);
|
||||
#if MI_PADDING_CHECK
|
||||
if (!mi_page_is_huge(page)) {
|
||||
uint8_t* fill = (uint8_t*)padding - delta;
|
||||
const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes
|
||||
for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; }
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
||||
return block;
|
||||
}
|
||||
|
||||
// extra entries for improved efficiency in `alloc-aligned.c` (and in `page.c:mi_malloc_generic`.
|
||||
extern void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept {
|
||||
return mi_page_malloc_zero(theap, page, size, zero, NULL);
|
||||
}
|
||||
|
||||
#if MI_GUARDED
|
||||
mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept;
|
||||
#endif
|
||||
|
||||
// main allocation primitives for small and generic allocation
|
||||
|
||||
// internal small size allocation
|
||||
static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept
|
||||
{
|
||||
mi_assert(theap != NULL);
|
||||
mi_assert(size <= MI_SMALL_SIZE_MAX);
|
||||
#if MI_DEBUG
|
||||
const uintptr_t tid = _mi_thread_id();
|
||||
mi_assert(theap->tld->thread_id == 0 || theap->tld->thread_id == tid); // theaps are thread local
|
||||
#endif
|
||||
#if (MI_PADDING || MI_GUARDED)
|
||||
if mi_unlikely(size == 0) { size = sizeof(void*); }
|
||||
#endif
|
||||
#if MI_GUARDED
|
||||
if mi_unlikely(mi_theap_malloc_use_guarded(theap,size)) {
|
||||
return _mi_theap_malloc_guarded(theap, size, zero);
|
||||
}
|
||||
#endif
|
||||
|
||||
// get page in constant time, and allocate from it
|
||||
mi_page_t* page = _mi_theap_get_free_small_page(theap, size + MI_PADDING_SIZE);
|
||||
void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, zero, usable);
|
||||
mi_track_malloc(p,size,zero);
|
||||
|
||||
#if MI_DEBUG>3
|
||||
if (p != NULL && zero) {
|
||||
mi_assert_expensive(mi_mem_is_zero(p, size));
|
||||
}
|
||||
#endif
|
||||
return p;
|
||||
}
|
||||
|
||||
// internal generic allocation
|
||||
static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept
|
||||
{
|
||||
#if MI_GUARDED
|
||||
#if MI_THEAP_INITASNULL
|
||||
if (theap!=NULL)
|
||||
#endif
|
||||
if (huge_alignment==0 && mi_theap_malloc_use_guarded(theap, size)) {
|
||||
return _mi_theap_malloc_guarded(theap, size, zero);
|
||||
}
|
||||
#endif
|
||||
#if !MI_THEAP_INITASNULL
|
||||
mi_assert(theap!=NULL);
|
||||
#endif
|
||||
mi_assert(theap==NULL || theap->tld->thread_id == 0 || theap->tld->thread_id == _mi_thread_id()); // theaps are thread local
|
||||
mi_assert((huge_alignment & 1)==0);
|
||||
void* const p = _mi_malloc_generic(theap, size + MI_PADDING_SIZE, (zero ? 1 : 0) | huge_alignment, usable); // note: size can overflow but it is detected in malloc_generic
|
||||
mi_track_malloc(p, size, zero);
|
||||
|
||||
#if MI_DEBUG>3
|
||||
if (p != NULL && zero) {
|
||||
mi_assert_expensive(mi_mem_is_zero(p, size));
|
||||
}
|
||||
#endif
|
||||
return p;
|
||||
}
|
||||
|
||||
// internal small allocation
|
||||
static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept {
|
||||
#if MI_THEAP_INITASNULL
|
||||
if (theap!=NULL) {
|
||||
return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);
|
||||
}
|
||||
else {
|
||||
return mi_theap_malloc_generic(theap, size, zero, 0, usable); // tailcall
|
||||
}
|
||||
#else
|
||||
return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
// allocate a small block
|
||||
mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept {
|
||||
return mi_theap_malloc_small_zero(theap, size, false, NULL);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept {
|
||||
return mi_theap_malloc_small(_mi_theap_default(), size);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {
|
||||
return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, false, NULL);
|
||||
}
|
||||
|
||||
// The main internal allocation functions
|
||||
static mi_decl_forceinline void* mi_theap_malloc_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept {
|
||||
// fast path for small objects
|
||||
if mi_likely(size <= MI_SMALL_SIZE_MAX) {
|
||||
mi_assert_internal(huge_alignment == 0);
|
||||
return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);
|
||||
}
|
||||
else {
|
||||
return mi_theap_malloc_generic(theap, size, zero, huge_alignment, usable);
|
||||
}
|
||||
}
|
||||
|
||||
extern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept {
|
||||
// fast path for small objects
|
||||
#if MI_THEAP_INITASNULL
|
||||
if mi_likely(theap!=NULL && size <= MI_SMALL_SIZE_MAX)
|
||||
#else
|
||||
if mi_likely(size <= MI_SMALL_SIZE_MAX)
|
||||
#endif
|
||||
{
|
||||
mi_assert_internal(huge_alignment == 0);
|
||||
return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);
|
||||
}
|
||||
else {
|
||||
return mi_theap_malloc_generic(theap, size, zero, huge_alignment, usable);
|
||||
}
|
||||
}
|
||||
|
||||
void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept {
|
||||
return _mi_theap_malloc_zero_ex(theap, size, zero, 0, usable);
|
||||
}
|
||||
|
||||
|
||||
// Main allocation functions
|
||||
|
||||
mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept {
|
||||
return _mi_theap_malloc_zero(theap, size, false, NULL);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept {
|
||||
return mi_theap_malloc(_mi_theap_default(), size);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {
|
||||
return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, false, 0, NULL);
|
||||
}
|
||||
|
||||
|
||||
// zero initialized small block
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept {
|
||||
return mi_theap_malloc_small_zero(_mi_theap_default(), size, true, NULL);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept {
|
||||
return mi_theap_malloc_small_zero(theap, size, true, NULL);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {
|
||||
return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, true, NULL);
|
||||
}
|
||||
|
||||
|
||||
mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept {
|
||||
return _mi_theap_malloc_zero(theap, size, true, NULL);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept {
|
||||
return _mi_theap_malloc_zero(_mi_theap_default(), size, true, NULL);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {
|
||||
return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, true, 0, NULL);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept {
|
||||
size_t total;
|
||||
if (mi_count_size_overflow(count,size,&total)) return NULL;
|
||||
return mi_theap_zalloc(theap,total);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept {
|
||||
return mi_theap_calloc(_mi_theap_default(),count,size);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept {
|
||||
size_t total;
|
||||
if (mi_count_size_overflow(count, size, &total)) return NULL;
|
||||
return mi_heap_zalloc(heap, total);
|
||||
}
|
||||
|
||||
// Return usable size
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* usable) mi_attr_noexcept {
|
||||
return mi_theap_malloc_small_zero(_mi_theap_default(), size, false, usable);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_theap_umalloc(mi_theap_t* theap, size_t size, size_t* usable) mi_attr_noexcept {
|
||||
return _mi_theap_malloc_zero_ex(theap, size, false, 0, usable);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_umalloc(size_t size, size_t* usable) mi_attr_noexcept {
|
||||
return mi_theap_umalloc(_mi_theap_default(), size, usable);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc(size_t size, size_t* usable) mi_attr_noexcept {
|
||||
return _mi_theap_malloc_zero_ex(_mi_theap_default(), size, true, 0, usable);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* usable) mi_attr_noexcept {
|
||||
size_t total;
|
||||
if (mi_count_size_overflow(count,size,&total)) return NULL;
|
||||
return mi_uzalloc(total, usable);
|
||||
}
|
||||
|
||||
// Uninitialized `calloc`
|
||||
static mi_decl_restrict void* mi_theap_mallocn(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept {
|
||||
size_t total;
|
||||
if (mi_count_size_overflow(count, size, &total)) return NULL;
|
||||
return mi_theap_malloc(theap, total);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept {
|
||||
return mi_theap_mallocn(_mi_theap_default(),count,size);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept {
|
||||
size_t total;
|
||||
if (mi_count_size_overflow(count, size, &total)) return NULL;
|
||||
return mi_heap_malloc(heap, total);
|
||||
}
|
||||
|
||||
|
||||
// Expand (or shrink) in place (or fail)
|
||||
void* mi_expand(void* p, size_t newsize) mi_attr_noexcept {
|
||||
#if MI_PADDING
|
||||
// we do not shrink/expand with padding enabled
|
||||
MI_UNUSED(p); MI_UNUSED(newsize);
|
||||
return NULL;
|
||||
#else
|
||||
if (p == NULL) return NULL;
|
||||
const mi_page_t* const page = mi_validate_ptr_page(p,"mi_expand");
|
||||
const size_t size = _mi_usable_size(p,page);
|
||||
if (newsize > size) return NULL;
|
||||
return p; // it fits
|
||||
#endif
|
||||
}
|
||||
|
||||
void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept {
|
||||
// if p == NULL then behave as malloc.
|
||||
// else if size == 0 then reallocate to a zero-sized block (and don't return NULL, just as mi_malloc(0)).
|
||||
// (this means that returning NULL always indicates an error, and `p` will not have been freed in that case.)
|
||||
const mi_page_t* page;
|
||||
size_t size;
|
||||
if (p==NULL) {
|
||||
page = NULL;
|
||||
size = 0;
|
||||
if (usable_pre!=NULL) { *usable_pre = 0; }
|
||||
}
|
||||
else {
|
||||
page = mi_validate_ptr_page(p,"mi_realloc");
|
||||
size = _mi_usable_size(p,page);
|
||||
if (usable_pre!=NULL) { *usable_pre = mi_page_usable_block_size(page); }
|
||||
}
|
||||
if mi_unlikely(newsize<=size && newsize>=(size/2) && newsize>0 // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0)
|
||||
&& mi_page_heap(page)==_mi_theap_heap(theap)) // and within the same heap
|
||||
{
|
||||
mi_assert_internal(p!=NULL);
|
||||
// todo: do not track as the usable size is still the same in the free; adjust potential padding?
|
||||
// mi_track_resize(p,size,newsize)
|
||||
// if (newsize < size) { mi_track_mem_noaccess((uint8_t*)p + newsize, size - newsize); }
|
||||
if (usable_post!=NULL) { *usable_post = mi_page_usable_block_size(page); }
|
||||
return p; // reallocation still fits and not more than 50% waste
|
||||
}
|
||||
void* newp = mi_theap_umalloc(theap,newsize,usable_post);
|
||||
if mi_likely(newp != NULL) {
|
||||
if (zero && newsize > size) {
|
||||
// also set last word in the previous allocation to zero to ensure any padding is zero-initialized
|
||||
const size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);
|
||||
_mi_memzero((uint8_t*)newp + start, newsize - start);
|
||||
}
|
||||
else if (newsize == 0) {
|
||||
((uint8_t*)newp)[0] = 0; // work around for applications that expect zero-reallocation to be zero initialized (issue #725)
|
||||
}
|
||||
if mi_likely(p != NULL) {
|
||||
const size_t copysize = (newsize > size ? size : newsize);
|
||||
mi_track_mem_defined(p,copysize); // _mi_useable_size may be too large for byte precise memory tracking..
|
||||
_mi_memcpy(newp, p, copysize);
|
||||
mi_free(p); // only free the original pointer if successful // todo: optimize since page is known?
|
||||
}
|
||||
}
|
||||
return newp;
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept {
|
||||
return _mi_theap_realloc_zero(theap, p, newsize, false, NULL, NULL);
|
||||
}
|
||||
|
||||
static void* mi_theap_reallocn(mi_theap_t* theap, void* p, size_t count, size_t size) mi_attr_noexcept {
|
||||
size_t total;
|
||||
if (mi_count_size_overflow(count, size, &total)) return NULL;
|
||||
return mi_theap_realloc(theap, p, total);
|
||||
}
|
||||
|
||||
|
||||
// Reallocate but free `p` on errors
|
||||
static void* mi_theap_reallocf(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept {
|
||||
void* newp = mi_theap_realloc(theap, p, newsize);
|
||||
if (newp==NULL && p!=NULL) mi_free(p);
|
||||
return newp;
|
||||
}
|
||||
|
||||
static void* mi_theap_rezalloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept {
|
||||
return _mi_theap_realloc_zero(theap, p, newsize, true, NULL, NULL);
|
||||
}
|
||||
|
||||
static void* mi_theap_recalloc(mi_theap_t* theap, void* p, size_t count, size_t size) mi_attr_noexcept {
|
||||
size_t total;
|
||||
if (mi_count_size_overflow(count, size, &total)) return NULL;
|
||||
return mi_theap_rezalloc(theap, p, total);
|
||||
}
|
||||
|
||||
|
||||
mi_decl_nodiscard void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept {
|
||||
return mi_theap_realloc(_mi_theap_default(),p,newsize);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept {
|
||||
return mi_theap_reallocn(_mi_theap_default(),p,count,size);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_urealloc(void* p, size_t newsize, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept {
|
||||
return _mi_theap_realloc_zero(_mi_theap_default(),p,newsize, false, usable_pre, usable_post);
|
||||
}
|
||||
|
||||
// Reallocate but free `p` on errors
|
||||
mi_decl_nodiscard void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept {
|
||||
return mi_theap_reallocf(_mi_theap_default(),p,newsize);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept {
|
||||
return mi_theap_rezalloc(_mi_theap_default(), p, newsize);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept {
|
||||
return mi_theap_recalloc(_mi_theap_default(), p, count, size);
|
||||
}
|
||||
|
||||
|
||||
mi_decl_nodiscard void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {
|
||||
return mi_theap_realloc(_mi_heap_theap(heap), p, newsize);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept {
|
||||
return mi_theap_reallocn(_mi_heap_theap(heap), p, count, size);
|
||||
}
|
||||
|
||||
// Reallocate but free `p` on errors
|
||||
mi_decl_nodiscard void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {
|
||||
return mi_theap_reallocf(_mi_heap_theap(heap), p, newsize);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {
|
||||
return mi_theap_rezalloc(_mi_heap_theap(heap), p, newsize);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept {
|
||||
return mi_theap_recalloc(_mi_heap_theap(heap), p, count, size);
|
||||
}
|
||||
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// strdup, strndup, and realpath
|
||||
// ------------------------------------------------------
|
||||
|
||||
// `strdup` using mi_malloc
|
||||
mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strdup(mi_theap_t* theap, const char* s) mi_attr_noexcept {
|
||||
if (s == NULL) return NULL;
|
||||
size_t len = _mi_strlen(s);
|
||||
if (len > MI_MAX_ALLOC_SIZE - 1) return NULL; // prevent overflow on len+1
|
||||
char* t = (char*)mi_theap_malloc(theap,len+1);
|
||||
if (t == NULL) return NULL;
|
||||
_mi_memcpy(t, s, len);
|
||||
t[len] = 0;
|
||||
return t;
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept {
|
||||
return mi_theap_strdup(_mi_theap_default(), s);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept {
|
||||
return mi_theap_strdup(_mi_heap_theap(heap), s);
|
||||
}
|
||||
|
||||
// `strndup` using mi_malloc
|
||||
mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strndup(mi_theap_t* theap, const char* s, size_t n) mi_attr_noexcept {
|
||||
if (s == NULL) return NULL;
|
||||
const size_t len = _mi_strnlen(s,n); // len <= n
|
||||
if (len > MI_MAX_ALLOC_SIZE - 1) return NULL; // prevent overflow on len+1
|
||||
char* t = (char*)mi_theap_malloc(theap, len+1);
|
||||
if (t == NULL) return NULL;
|
||||
_mi_memcpy(t, s, len);
|
||||
t[len] = 0;
|
||||
return t;
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept {
|
||||
return mi_theap_strndup(_mi_theap_default(),s,n);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept {
|
||||
return mi_theap_strndup(_mi_heap_theap(heap), s, n);
|
||||
}
|
||||
|
||||
#ifndef __wasi__
|
||||
// `realpath` using mi_malloc
|
||||
#ifdef _WIN32
|
||||
#ifndef PATH_MAX
|
||||
#define PATH_MAX MAX_PATH
|
||||
#endif
|
||||
|
||||
mi_decl_nodiscard static mi_decl_restrict char* mi_theap_realpath(mi_theap_t* theap, const char* fname, char* resolved_name) mi_attr_noexcept {
|
||||
// todo: use GetFullPathNameW to allow longer file names
|
||||
char buf[PATH_MAX];
|
||||
DWORD res = GetFullPathNameA(fname, PATH_MAX, (resolved_name == NULL ? buf : resolved_name), NULL);
|
||||
if (res == 0) {
|
||||
errno = GetLastError(); return NULL;
|
||||
}
|
||||
else if (res > PATH_MAX) {
|
||||
errno = EINVAL; return NULL;
|
||||
}
|
||||
else if (resolved_name != NULL) {
|
||||
return resolved_name;
|
||||
}
|
||||
else {
|
||||
return mi_theap_strndup(theap, buf, PATH_MAX);
|
||||
}
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
#include <unistd.h> // pathconf
|
||||
|
||||
static size_t mi_path_max(void) {
|
||||
static _Atomic(size_t) path_max = 0;
|
||||
size_t pmax = mi_atomic_load_acquire(&path_max);
|
||||
if (pmax == 0) {
|
||||
long m = 0;
|
||||
#ifdef _PC_PATH_MAX
|
||||
m = pathconf("/",_PC_PATH_MAX);
|
||||
#endif
|
||||
if (m <= 0) pmax = 4096; // guess
|
||||
else if (m < 256) pmax = 256; // at least 256
|
||||
else if (m > 64*1024) pmax = 64*1024; // at most 64 KiB
|
||||
else pmax = m;
|
||||
size_t expected = 0;
|
||||
mi_atomic_cas_strong_acq_rel(&path_max, &expected, pmax);
|
||||
}
|
||||
return pmax;
|
||||
}
|
||||
|
||||
char* mi_theap_realpath(mi_theap_t* theap, const char* fname, char* resolved_name) mi_attr_noexcept {
|
||||
if (resolved_name != NULL) {
|
||||
return realpath(fname,resolved_name);
|
||||
}
|
||||
else {
|
||||
/*
|
||||
char* rname = realpath(fname, NULL);
|
||||
if (rname == NULL) return NULL;
|
||||
char* result = mi_heap_strdup(heap, rname);
|
||||
mi_cfree(rname); // note: may leak the original pointer if allocated internally with the system allocator
|
||||
// note: with ASAN realpath is intercepted and mi_cfree may leak the returned pointer :-(
|
||||
return result;
|
||||
*/
|
||||
const size_t n = mi_path_max();
|
||||
char* const buf = (char*)mi_zalloc(n+1);
|
||||
if (buf == NULL) {
|
||||
errno = ENOMEM;
|
||||
return NULL;
|
||||
}
|
||||
char* rname = realpath(fname,buf);
|
||||
char* result = mi_theap_strndup(theap,rname,n); // ok if `rname==NULL`
|
||||
mi_free(buf);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept {
|
||||
return mi_theap_realpath(_mi_theap_default(),fname,resolved_name);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept {
|
||||
return mi_theap_realpath(_mi_heap_theap(heap), fname, resolved_name);
|
||||
}
|
||||
#endif
|
||||
|
||||
/*-------------------------------------------------------
|
||||
C++ new and new_aligned
|
||||
The standard requires calling into `get_new_handler` and
|
||||
throwing the bad_alloc exception on failure. If we compile
|
||||
with a C++ compiler we can implement this precisely. If we
|
||||
use a C compiler we cannot throw a `bad_alloc` exception
|
||||
but we call `exit` instead (i.e. not returning).
|
||||
-------------------------------------------------------*/
|
||||
|
||||
#ifdef __cplusplus
|
||||
#include <new>
|
||||
static bool mi_try_new_handler(bool nothrow) {
|
||||
#if defined(_MSC_VER) || (__cplusplus >= 201103L)
|
||||
std::new_handler h = std::get_new_handler();
|
||||
#else
|
||||
std::new_handler h = std::set_new_handler();
|
||||
std::set_new_handler(h);
|
||||
#endif
|
||||
if (h==NULL) {
|
||||
_mi_error_message(ENOMEM, "out of memory in 'new'");
|
||||
#if defined(_CPPUNWIND) || defined(__cpp_exceptions) // exceptions are not always enabled
|
||||
if (!nothrow) {
|
||||
throw std::bad_alloc();
|
||||
}
|
||||
#else
|
||||
MI_UNUSED(nothrow);
|
||||
#endif
|
||||
return false;
|
||||
}
|
||||
else {
|
||||
h();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
#else
|
||||
typedef void (*std_new_handler_t)(void);
|
||||
|
||||
#if (defined(__GNUC__) || (defined(__clang__) && !defined(_MSC_VER))) // exclude clang-cl, see issue #631
|
||||
std_new_handler_t __attribute__((weak)) _ZSt15get_new_handlerv(void) {
|
||||
return NULL;
|
||||
}
|
||||
static std_new_handler_t mi_get_new_handler(void) {
|
||||
return _ZSt15get_new_handlerv();
|
||||
}
|
||||
#else
|
||||
// note: on windows we could dynamically link to `?get_new_handler@std@@YAP6AXXZXZ`.
|
||||
static std_new_handler_t mi_get_new_handler(void) {
|
||||
return NULL;
|
||||
}
|
||||
#endif
|
||||
|
||||
static bool mi_try_new_handler(bool nothrow) {
|
||||
std_new_handler_t h = mi_get_new_handler();
|
||||
if (h==NULL) {
|
||||
_mi_error_message(ENOMEM, "out of memory in 'new'");
|
||||
if (!nothrow) {
|
||||
abort(); // cannot throw in plain C, use abort
|
||||
}
|
||||
return false;
|
||||
}
|
||||
else {
|
||||
h();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
static mi_decl_noinline void* mi_theap_try_new(mi_theap_t* theap, size_t size, bool nothrow ) {
|
||||
void* p = NULL;
|
||||
while(p == NULL && mi_try_new_handler(nothrow)) {
|
||||
p = mi_theap_malloc(theap,size);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
static mi_decl_noinline void* mi_try_new(size_t size, bool nothrow) {
|
||||
return mi_theap_try_new(_mi_theap_default(), size, nothrow);
|
||||
}
|
||||
|
||||
static mi_decl_noinline void* mi_heap_try_new(mi_heap_t* heap, size_t size, bool nothrow) {
|
||||
return mi_theap_try_new(_mi_heap_theap(heap), size, nothrow);
|
||||
}
|
||||
|
||||
|
||||
mi_decl_nodiscard static mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* theap, size_t size) {
|
||||
void* p = mi_theap_malloc(theap,size);
|
||||
if mi_unlikely(p == NULL) return mi_theap_try_new(theap, size, false);
|
||||
return p;
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_new(size_t size) {
|
||||
return mi_theap_alloc_new(_mi_theap_default(), size);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) {
|
||||
void* p = mi_heap_malloc(heap, size);
|
||||
if mi_unlikely(p == NULL) return mi_heap_try_new(heap, size, false);
|
||||
return p;
|
||||
}
|
||||
|
||||
|
||||
mi_decl_nodiscard static mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* theap, size_t count, size_t size) {
|
||||
size_t total;
|
||||
if mi_unlikely(mi_count_size_overflow(count, size, &total)) {
|
||||
mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc
|
||||
return NULL;
|
||||
}
|
||||
else {
|
||||
return mi_theap_alloc_new(theap,total);
|
||||
}
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_new_n(size_t count, size_t size) {
|
||||
return mi_theap_alloc_new_n(_mi_theap_default(), count, size);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) {
|
||||
return mi_theap_alloc_new_n(_mi_heap_theap(heap), count, size);
|
||||
}
|
||||
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept {
|
||||
void* p = mi_malloc(size);
|
||||
if mi_unlikely(p == NULL) return mi_try_new(size, true);
|
||||
return p;
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) {
|
||||
void* p;
|
||||
do {
|
||||
p = mi_malloc_aligned(size, alignment);
|
||||
}
|
||||
while(p == NULL && mi_try_new_handler(false));
|
||||
return p;
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept {
|
||||
void* p;
|
||||
do {
|
||||
p = mi_malloc_aligned(size, alignment);
|
||||
}
|
||||
while(p == NULL && mi_try_new_handler(true));
|
||||
return p;
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_new_realloc(void* p, size_t newsize) {
|
||||
void* q;
|
||||
do {
|
||||
q = mi_realloc(p, newsize);
|
||||
} while (q == NULL && mi_try_new_handler(false));
|
||||
return q;
|
||||
}
|
||||
|
||||
mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) {
|
||||
size_t total;
|
||||
if mi_unlikely(mi_count_size_overflow(newcount, size, &total)) {
|
||||
mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc
|
||||
return NULL;
|
||||
}
|
||||
else {
|
||||
return mi_new_realloc(p, total);
|
||||
}
|
||||
}
|
||||
|
||||
#if MI_GUARDED
|
||||
// We always allocate a guarded allocation at an offset (`mi_page_has_interior_pointers` will be true).
|
||||
// We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`)
|
||||
// and the first word to `~0` for guarded allocations to have a correct `mi_usable_size`
|
||||
|
||||
static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size) {
|
||||
// TODO: we can still make padding work by moving it out of the guard page area
|
||||
mi_page_t* const page = _mi_ptr_page(block);
|
||||
mi_page_set_has_interior_pointers(page, true);
|
||||
block->next = MI_BLOCK_TAG_GUARDED;
|
||||
|
||||
// set guard page at the end of the block
|
||||
const size_t block_size = mi_page_block_size(page); // must use `block_size` to match `mi_free_local`
|
||||
const size_t os_page_size = _mi_os_page_size();
|
||||
mi_assert_internal(block_size >= obj_size + os_page_size + sizeof(mi_block_t));
|
||||
if (block_size < obj_size + os_page_size + sizeof(mi_block_t)) {
|
||||
// should never happen
|
||||
mi_free(block);
|
||||
return NULL;
|
||||
}
|
||||
uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size;
|
||||
// note: the alignment of the guard page relies on blocks being os_page_size aligned which
|
||||
// is ensured in `mi_arena_page_alloc_fresh`.
|
||||
mi_assert_internal(_mi_is_aligned(block, os_page_size));
|
||||
mi_assert_internal(_mi_is_aligned(guard_page, os_page_size));
|
||||
if (!page->memid.is_pinned && _mi_is_aligned(guard_page, os_page_size)) {
|
||||
const bool ok = _mi_os_protect(guard_page, os_page_size);
|
||||
if mi_unlikely(!ok) {
|
||||
_mi_warning_message("failed to set a guard page behind an object (object %p of size %zu)\n", block, block_size);
|
||||
}
|
||||
}
|
||||
else {
|
||||
_mi_warning_message("unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\n", block, block_size);
|
||||
}
|
||||
|
||||
// align pointer just in front of the guard page
|
||||
size_t offset = block_size - os_page_size - obj_size;
|
||||
mi_assert_internal(offset > sizeof(mi_block_t));
|
||||
if (offset > MI_PAGE_MAX_OVERALLOC_ALIGN) {
|
||||
// give up to place it right in front of the guard page if the offset is too large for unalignment
|
||||
offset = MI_PAGE_MAX_OVERALLOC_ALIGN;
|
||||
}
|
||||
uint8_t* const p = (uint8_t*)block + offset;
|
||||
mi_assert_internal(p == guard_page - obj_size);
|
||||
mi_track_align(block, p, offset, obj_size);
|
||||
mi_track_mem_defined(block, sizeof(mi_block_t));
|
||||
return p;
|
||||
}
|
||||
|
||||
mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept
|
||||
{
|
||||
// allocate multiple of page size ending in a guard page
|
||||
// ensure minimal alignment requirement?
|
||||
if mi_unlikely(size >= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { // check up front so the `req_size` won't overflow
|
||||
_mi_error_message(EOVERFLOW, "(guarded) allocation request is too large (%zu bytes)\n", size);
|
||||
return NULL;
|
||||
}
|
||||
const size_t os_page_size = _mi_os_page_size();
|
||||
const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE));
|
||||
const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE);
|
||||
const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size);
|
||||
mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(theap, req_size, 0 /* don't zero */, NULL);
|
||||
if (block==NULL) return NULL;
|
||||
void* const p = mi_block_ptr_set_guarded(block, obj_size);
|
||||
if (p == NULL) return p;
|
||||
if (zero) {
|
||||
_mi_memzero_aligned(p,obj_size); // we have to zero here as padding might have written here (if the blocksize > reqsize + os_page_size)
|
||||
}
|
||||
|
||||
// stats
|
||||
mi_track_malloc(p, obj_size, zero);
|
||||
if (!mi_theap_is_initialized(theap)) { theap = _mi_theap_default(); }
|
||||
mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1);
|
||||
#if MI_STAT>1
|
||||
// adjust stats to only count the allocated size of the block (and not the guard page)
|
||||
mi_theap_stat_adjust_decrease(theap, malloc_requested, req_size);
|
||||
mi_theap_stat_increase(theap, malloc_requested, size);
|
||||
#endif
|
||||
#if MI_DEBUG>3
|
||||
if (zero) {
|
||||
mi_assert_expensive(mi_mem_is_zero(p, size));
|
||||
}
|
||||
#endif
|
||||
return p;
|
||||
}
|
||||
#endif
|
||||
|
||||
// ------------------------------------------------------
|
||||
// ensure explicit external inline definitions are emitted!
|
||||
// ------------------------------------------------------
|
||||
|
||||
#ifdef __cplusplus
|
||||
void* _mi_externs[] = {
|
||||
(void*)&_mi_page_malloc_zero,
|
||||
(void*)&_mi_theap_malloc_zero,
|
||||
(void*)&_mi_theap_malloc_zero_ex,
|
||||
(void*)&mi_theap_malloc,
|
||||
(void*)&mi_theap_zalloc,
|
||||
(void*)&mi_theap_malloc_small,
|
||||
(void*)&mi_malloc,
|
||||
(void*)&mi_malloc_small,
|
||||
(void*)&mi_zalloc,
|
||||
(void*)&mi_zalloc_small,
|
||||
(void*)&mi_heap_malloc,
|
||||
(void*)&mi_heap_malloc_small,
|
||||
(void*)&mi_malloc_aligned
|
||||
// (void*)&mi_theap_alloc_new,
|
||||
// (void*)&mi_theap_alloc_new_n
|
||||
};
|
||||
#endif
|
||||
@@ -0,0 +1,179 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2019-2024, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
We have a special "mini" allocator just for allocation of meta-data like
|
||||
the theap (`mi_theap_t`) or thread-local data (`mi_tld_t`).
|
||||
|
||||
We reuse the bitmap of the arena's for allocation of 64b blocks inside
|
||||
an arena slice (64KiB).
|
||||
We always ensure that meta data is zero'd (we zero on `free`)
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc/internal.h"
|
||||
#include "bitmap.h"
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Meta data allocation
|
||||
----------------------------------------------------------- */
|
||||
|
||||
#define MI_META_PAGE_SIZE MI_ARENA_SLICE_SIZE
|
||||
#define MI_META_PAGE_ALIGN MI_ARENA_SLICE_ALIGN
|
||||
|
||||
// large enough such that META_MAX_SIZE > 4k (even on 32-bit)
|
||||
#define MI_META_BLOCK_SIZE (1 << (16 - MI_BCHUNK_BITS_SHIFT)) // 128 on 64-bit
|
||||
#define MI_META_BLOCK_ALIGN MI_META_BLOCK_SIZE
|
||||
#define MI_META_BLOCKS_PER_PAGE (MI_META_PAGE_SIZE / MI_META_BLOCK_SIZE) // 512
|
||||
#define MI_META_MAX_SIZE (MI_BCHUNK_SIZE * MI_META_BLOCK_SIZE)
|
||||
|
||||
#if MI_META_MAX_SIZE <= 4096
|
||||
#error "max meta object size should be at least 4KiB"
|
||||
#endif
|
||||
|
||||
typedef struct mi_meta_page_s {
|
||||
_Atomic(struct mi_meta_page_s*) next; // a linked list of meta-data pages (never released)
|
||||
mi_memid_t memid; // provenance of the meta-page memory itself
|
||||
mi_bbitmap_t blocks_free; // a small bitmap with 1 bit per block.
|
||||
} mi_meta_page_t;
|
||||
|
||||
static mi_decl_cache_align _Atomic(mi_meta_page_t*) mi_meta_pages = MI_ATOMIC_VAR_INIT(NULL);
|
||||
|
||||
|
||||
#if MI_DEBUG > 1
|
||||
static mi_meta_page_t* mi_meta_page_of_ptr(void* p, size_t* block_idx) {
|
||||
mi_meta_page_t* mpage = (mi_meta_page_t*)((uint8_t*)_mi_align_down_ptr(p,MI_META_PAGE_ALIGN) + _mi_os_secure_guard_page_size());
|
||||
if (block_idx != NULL) {
|
||||
*block_idx = ((uint8_t*)p - (uint8_t*)mpage) / MI_META_BLOCK_SIZE;
|
||||
}
|
||||
return mpage;
|
||||
}
|
||||
#endif
|
||||
|
||||
static mi_meta_page_t* mi_meta_page_next( mi_meta_page_t* mpage ) {
|
||||
return mi_atomic_load_ptr_acquire(mi_meta_page_t, &mpage->next);
|
||||
}
|
||||
|
||||
static void* mi_meta_block_start( mi_meta_page_t* mpage, size_t block_idx ) {
|
||||
mi_assert_internal(_mi_is_aligned((uint8_t*)mpage - _mi_os_secure_guard_page_size(), MI_META_PAGE_ALIGN));
|
||||
mi_assert_internal(block_idx < MI_META_BLOCKS_PER_PAGE);
|
||||
void* p = ((uint8_t*)mpage - _mi_os_secure_guard_page_size() + (block_idx * MI_META_BLOCK_SIZE));
|
||||
mi_assert_internal(mpage == mi_meta_page_of_ptr(p,NULL));
|
||||
return p;
|
||||
}
|
||||
|
||||
// allocate a fresh meta page and add it to the global list.
|
||||
static mi_meta_page_t* mi_meta_page_zalloc(void) {
|
||||
// allocate a fresh arena slice
|
||||
// note: careful with _mi_subproc as it may recurse into mi_tld and meta_page_zalloc again.. (same with _mi_os_numa_node()...)
|
||||
mi_memid_t memid;
|
||||
uint8_t* base = (uint8_t*)_mi_arenas_alloc_aligned(mi_heap_main(), MI_META_PAGE_SIZE, MI_META_PAGE_ALIGN, 0,
|
||||
true /* commit*/, (MI_SECURE==0) /* allow large? */,
|
||||
NULL /* req arena */, 0 /* thread_seq */, -1 /* numa node */, &memid);
|
||||
if (base == NULL) return NULL;
|
||||
mi_assert_internal(_mi_is_aligned(base,MI_META_PAGE_ALIGN));
|
||||
if (!memid.initially_zero) {
|
||||
_mi_memzero_aligned(base, MI_ARENA_SLICE_SIZE);
|
||||
}
|
||||
|
||||
// guard pages
|
||||
#if MI_SECURE >= 1
|
||||
_mi_os_secure_guard_page_set_at(base, memid);
|
||||
_mi_os_secure_guard_page_set_before(base + MI_META_PAGE_SIZE, memid);
|
||||
#endif
|
||||
|
||||
// initialize the page and free block bitmap
|
||||
mi_meta_page_t* mpage = (mi_meta_page_t*)(base + _mi_os_secure_guard_page_size());
|
||||
mpage->memid = memid;
|
||||
mi_bbitmap_init(&mpage->blocks_free, MI_META_BLOCKS_PER_PAGE, true /* already_zero */);
|
||||
const size_t mpage_size = offsetof(mi_meta_page_t,blocks_free) + mi_bbitmap_size(MI_META_BLOCKS_PER_PAGE, NULL);
|
||||
const size_t info_blocks = _mi_divide_up(mpage_size,MI_META_BLOCK_SIZE);
|
||||
const size_t guard_blocks = _mi_divide_up(_mi_os_secure_guard_page_size(), MI_META_BLOCK_SIZE);
|
||||
mi_assert_internal(info_blocks + 2*guard_blocks < MI_META_BLOCKS_PER_PAGE);
|
||||
mi_bbitmap_unsafe_setN(&mpage->blocks_free, info_blocks + guard_blocks, MI_META_BLOCKS_PER_PAGE - info_blocks - 2*guard_blocks);
|
||||
|
||||
// push atomically in front of the meta page list
|
||||
// (note: there is no ABA issue since we never free meta-pages)
|
||||
mi_meta_page_t* old = mi_atomic_load_ptr_acquire(mi_meta_page_t,&mi_meta_pages);
|
||||
do {
|
||||
mi_atomic_store_ptr_release(mi_meta_page_t, &mpage->next, old);
|
||||
} while(!mi_atomic_cas_ptr_weak_acq_rel(mi_meta_page_t,&mi_meta_pages,&old,mpage));
|
||||
return mpage;
|
||||
}
|
||||
|
||||
|
||||
// allocate meta-data
|
||||
mi_decl_noinline void* _mi_meta_zalloc( size_t size, mi_memid_t* pmemid )
|
||||
{
|
||||
mi_assert_internal(pmemid != NULL);
|
||||
size = _mi_align_up(size,MI_META_BLOCK_SIZE);
|
||||
if (size == 0 || size > MI_META_MAX_SIZE) return NULL;
|
||||
const size_t block_count = _mi_divide_up(size,MI_META_BLOCK_SIZE);
|
||||
mi_assert_internal(block_count > 0 && block_count < MI_BCHUNK_BITS);
|
||||
mi_meta_page_t* mpage0 = mi_atomic_load_ptr_acquire(mi_meta_page_t,&mi_meta_pages);
|
||||
mi_meta_page_t* mpage = mpage0;
|
||||
while (mpage != NULL) {
|
||||
size_t block_idx;
|
||||
if (mi_bbitmap_try_find_and_clearN(&mpage->blocks_free, 0, block_count, &block_idx)) {
|
||||
// found and claimed `block_count` blocks
|
||||
*pmemid = _mi_memid_create_meta(mpage, block_idx, block_count);
|
||||
return mi_meta_block_start(mpage,block_idx);
|
||||
}
|
||||
else {
|
||||
mpage = mi_meta_page_next(mpage);
|
||||
}
|
||||
}
|
||||
// failed to find space in existing pages
|
||||
if (mi_atomic_load_ptr_acquire(mi_meta_page_t,&mi_meta_pages) != mpage0) {
|
||||
// the page list was updated by another thread in the meantime, retry
|
||||
return _mi_meta_zalloc(size,pmemid);
|
||||
}
|
||||
// otherwise, allocate a fresh metapage and try once more
|
||||
mpage = mi_meta_page_zalloc();
|
||||
if (mpage != NULL) {
|
||||
size_t block_idx;
|
||||
if (mi_bbitmap_try_find_and_clearN(&mpage->blocks_free, 0, block_count, &block_idx)) {
|
||||
// found and claimed `block_count` blocks
|
||||
*pmemid = _mi_memid_create_meta(mpage, block_idx, block_count);
|
||||
return mi_meta_block_start(mpage,block_idx);
|
||||
}
|
||||
}
|
||||
// if all this failed, allocate from the OS
|
||||
return _mi_os_alloc(size, pmemid);
|
||||
}
|
||||
|
||||
// free meta-data
|
||||
mi_decl_noinline void _mi_meta_free(void* p, size_t size, mi_memid_t memid) {
|
||||
if (p==NULL) return;
|
||||
if (memid.memkind == MI_MEM_META) {
|
||||
mi_assert_internal(_mi_divide_up(size, MI_META_BLOCK_SIZE) == memid.mem.meta.block_count);
|
||||
const size_t block_count = memid.mem.meta.block_count;
|
||||
const size_t block_idx = memid.mem.meta.block_index;
|
||||
mi_meta_page_t* mpage = (mi_meta_page_t*)memid.mem.meta.meta_page;
|
||||
mi_assert_internal(mi_meta_page_of_ptr(p,NULL) == mpage);
|
||||
mi_assert_internal(block_idx + block_count <= MI_META_BLOCKS_PER_PAGE);
|
||||
mi_assert_internal(mi_bbitmap_is_clearN(&mpage->blocks_free, block_idx, block_count));
|
||||
// we zero on free (and on the initial page allocation) so we don't need a "dirty" map
|
||||
_mi_memzero_aligned(mi_meta_block_start(mpage, block_idx), block_count*MI_META_BLOCK_SIZE);
|
||||
mi_bbitmap_setN(&mpage->blocks_free, block_idx, block_count);
|
||||
}
|
||||
else {
|
||||
_mi_arenas_free(p,size,memid);
|
||||
}
|
||||
}
|
||||
|
||||
// used for debug output
|
||||
bool _mi_meta_is_meta_page(void* p)
|
||||
{
|
||||
mi_meta_page_t* mpage0 = mi_atomic_load_ptr_acquire(mi_meta_page_t, &mi_meta_pages);
|
||||
mi_meta_page_t* mpage = mpage0;
|
||||
while (mpage != NULL) {
|
||||
if ((void*)mpage == p) return true;
|
||||
mpage = mi_meta_page_next(mpage);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,343 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2019-2024 Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
Concurrent bitmap that can set/reset sequences of bits atomically
|
||||
---------------------------------------------------------------------------- */
|
||||
#pragma once
|
||||
#ifndef MI_BITMAP_H
|
||||
#define MI_BITMAP_H
|
||||
|
||||
/* --------------------------------------------------------------------------------
|
||||
Atomic bitmaps with release/acquire guarantees:
|
||||
|
||||
`mi_bfield_t`: is a single machine word that can efficiently be bit counted (usually `size_t`)
|
||||
each bit usually represents a single MI_ARENA_SLICE_SIZE in an arena (64 KiB).
|
||||
We need 16K bits to represent a 1GiB arena.
|
||||
|
||||
`mi_bchunk_t`: a chunk of bfield's of a total of MI_BCHUNK_BITS (= 512 on 64-bit, 256 on 32-bit)
|
||||
allocations never span across chunks -- so MI_ARENA_MAX_OBJ_SIZE is the number
|
||||
of bits in a chunk times the MI_ARENA_SLICE_SIZE (512 * 64KiB = 32 MiB).
|
||||
These chunks are cache-aligned and we can use AVX2/AVX512/NEON/SVE/SVE2/etc. instructions
|
||||
to scan for bits (perhaps) more efficiently.
|
||||
|
||||
We allocate byte-sized ranges aligned to bytes in the bfield, and bfield-sized
|
||||
ranges aligned to a bfield.
|
||||
|
||||
Searching linearly through the chunks would be too slow (16K bits per GiB).
|
||||
Instead we add a "chunkmap" to do a two-level search (more or less a btree of depth 2).
|
||||
|
||||
`mi_bchunkmap_t` (== `mi_bchunk_t`): for each chunk we track if it has (potentially) any bit set.
|
||||
The chunkmap has 1 bit per chunk that is set if the chunk potentially has a bit set.
|
||||
This is used to avoid scanning every chunk. (and thus strictly an optimization)
|
||||
It is conservative: it is fine to set a bit in the chunk map even if the chunk turns out
|
||||
to have no bits set. It is also allowed to briefly have a clear bit even if the
|
||||
chunk has bits set -- as long as we guarantee that the bit will be set later on;
|
||||
(this allows us to set the chunkmap bit right after we set a bit in the corresponding chunk).
|
||||
|
||||
However, when we clear a bit in a chunk, and the chunk is indeed all clear, we
|
||||
cannot safely clear the bit corresponding to the chunk in the chunkmap since it
|
||||
may race with another thread setting a bit in the same chunk. Therefore, when
|
||||
clearing, we first test if a chunk is clear, then clear the chunkmap bit, and
|
||||
then test again to catch any set bits that we may have missed.
|
||||
|
||||
Since the chunkmap may thus be briefly out-of-sync, this means that we may sometimes
|
||||
not find a free page even though it's there (but we accept this as we avoid taking
|
||||
full locks). (Another way to do this is to use an epoch but we like to avoid that complexity
|
||||
for now).
|
||||
|
||||
`mi_bitmap_t`: a bitmap with N chunks. A bitmap has a chunkmap of MI_BCHUNK_BITS (512)
|
||||
and thus has at most 512 chunks (=2^18 bits x 64 KiB slices = 16 GiB max arena size).
|
||||
The minimum is 1 chunk which is a 32 MiB arena.
|
||||
|
||||
For now, the implementation assumes MI_HAS_FAST_BITSCAN and uses trailing-zero-count
|
||||
and pop-count (but we think it can be adapted work reasonably well on older hardware too)
|
||||
--------------------------------------------------------------------------------------------- */
|
||||
|
||||
// A word-size bit field.
|
||||
typedef size_t mi_bfield_t;
|
||||
|
||||
#define MI_BFIELD_BITS_SHIFT (MI_SIZE_SHIFT+3)
|
||||
#define MI_BFIELD_BITS (1 << MI_BFIELD_BITS_SHIFT)
|
||||
#define MI_BFIELD_SIZE (MI_BFIELD_BITS/8)
|
||||
#define MI_BFIELD_LO_BIT8 (((~(mi_bfield_t)0))/0xFF) // 0x01010101 ..
|
||||
#define MI_BFIELD_HI_BIT8 (MI_BFIELD_LO_BIT8 << 7) // 0x80808080 ..
|
||||
|
||||
#define MI_BCHUNK_SIZE (MI_BCHUNK_BITS / 8)
|
||||
#define MI_BCHUNK_FIELDS (MI_BCHUNK_BITS / MI_BFIELD_BITS) // 8 on both 64- and 32-bit
|
||||
|
||||
|
||||
// some compiler (msvc in C mode) cannot have expressions in the alignment attribute
|
||||
#if MI_BCHUNK_SIZE==64
|
||||
#define mi_decl_bchunk_align mi_decl_align(64)
|
||||
#elif MI_BCHUNK_SIZE==32
|
||||
#define mi_decl_bchunk_align mi_decl_align(32)
|
||||
#else
|
||||
#define mi_decl_bchunk_align mi_decl_align(MI_BCHUNK_SIZE)
|
||||
#endif
|
||||
|
||||
|
||||
// A bitmap chunk contains 512 bits on 64-bit (256 on 32-bit)
|
||||
typedef mi_decl_bchunk_align struct mi_bchunk_s {
|
||||
_Atomic(mi_bfield_t) bfields[MI_BCHUNK_FIELDS];
|
||||
} mi_bchunk_t;
|
||||
|
||||
|
||||
// The chunkmap has one bit per corresponding chunk that is set if the chunk potentially has bits set.
|
||||
// The chunkmap is itself a chunk.
|
||||
typedef mi_bchunk_t mi_bchunkmap_t;
|
||||
|
||||
#define MI_BCHUNKMAP_BITS MI_BCHUNK_BITS
|
||||
|
||||
#define MI_BITMAP_MAX_CHUNK_COUNT (MI_BCHUNKMAP_BITS)
|
||||
#define MI_BITMAP_MIN_CHUNK_COUNT (1)
|
||||
#if MI_SIZE_BITS > 32
|
||||
#define MI_BITMAP_DEFAULT_CHUNK_COUNT (64) // 2 GiB on 64-bit -- this is for the page map
|
||||
#else
|
||||
#define MI_BITMAP_DEFAULT_CHUNK_COUNT (1)
|
||||
#endif
|
||||
#define MI_BITMAP_MAX_BIT_COUNT (MI_BITMAP_MAX_CHUNK_COUNT * MI_BCHUNK_BITS) // 16 GiB arena
|
||||
#define MI_BITMAP_MIN_BIT_COUNT (MI_BITMAP_MIN_CHUNK_COUNT * MI_BCHUNK_BITS) // 32 MiB arena
|
||||
#define MI_BITMAP_DEFAULT_BIT_COUNT (MI_BITMAP_DEFAULT_CHUNK_COUNT * MI_BCHUNK_BITS) // 2 GiB arena
|
||||
|
||||
|
||||
// An atomic bitmap
|
||||
typedef mi_decl_bchunk_align struct mi_bitmap_s {
|
||||
_Atomic(size_t) chunk_count; // total count of chunks (0 < N <= MI_BCHUNKMAP_BITS)
|
||||
size_t _padding[MI_BCHUNK_SIZE/MI_SIZE_SIZE - 1]; // suppress warning on msvc
|
||||
mi_bchunkmap_t chunkmap;
|
||||
mi_bchunk_t chunks[MI_BITMAP_DEFAULT_CHUNK_COUNT]; // usually dynamic MI_BITMAP_MAX_CHUNK_COUNT
|
||||
} mi_bitmap_t;
|
||||
|
||||
|
||||
static inline size_t mi_bitmap_chunk_count(const mi_bitmap_t* bitmap) {
|
||||
return mi_atomic_load_relaxed(&((mi_bitmap_t*)bitmap)->chunk_count);
|
||||
}
|
||||
|
||||
static inline size_t mi_bitmap_max_bits(const mi_bitmap_t* bitmap) {
|
||||
return (mi_bitmap_chunk_count(bitmap) * MI_BCHUNK_BITS);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* --------------------------------------------------------------------------------
|
||||
Atomic bitmap operations
|
||||
-------------------------------------------------------------------------------- */
|
||||
|
||||
// Many operations are generic over setting or clearing the bit sequence: we use `mi_xset_t` for this (true if setting, false if clearing)
|
||||
typedef bool mi_xset_t;
|
||||
#define MI_BIT_SET (true)
|
||||
#define MI_BIT_CLEAR (false)
|
||||
|
||||
|
||||
// Required size of a bitmap to represent `bit_count` bits.
|
||||
size_t mi_bitmap_size(size_t bit_count, size_t* chunk_count);
|
||||
|
||||
// Initialize a bitmap to all clear; avoid a mem_zero if `already_zero` is true
|
||||
// returns the size of the bitmap.
|
||||
size_t mi_bitmap_init(mi_bitmap_t* bitmap, size_t bit_count, bool already_zero);
|
||||
|
||||
// Set/clear a sequence of `n` bits in the bitmap (and can cross chunks).
|
||||
// Not atomic so only use if still local to a thread.
|
||||
void mi_bitmap_unsafe_setN(mi_bitmap_t* bitmap, size_t idx, size_t n);
|
||||
|
||||
|
||||
// Set a bit in the bitmap; returns `true` if it atomically transitioned from 0 to 1
|
||||
bool mi_bitmap_set(mi_bitmap_t* bitmap, size_t idx);
|
||||
|
||||
// Clear a bit in the bitmap; returns `true` if it atomically transitioned from 1 to 0
|
||||
bool mi_bitmap_clear(mi_bitmap_t* bitmap, size_t idx);
|
||||
|
||||
// Set a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from all 0's to 1's
|
||||
// If `already_set` is not NULL, it is set to count of bits were already all set.
|
||||
// (this is used for correct statistics if commiting over a partially committed area)
|
||||
bool mi_bitmap_setN(mi_bitmap_t* bitmap, size_t idx, size_t n, size_t* already_set);
|
||||
|
||||
// Clear a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from all 1's to 0's
|
||||
bool mi_bitmap_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n);
|
||||
|
||||
|
||||
// Is a sequence of n bits already all set/cleared?
|
||||
bool mi_bitmap_is_xsetN(mi_xset_t set, mi_bitmap_t* bitmap, size_t idx, size_t n);
|
||||
|
||||
// Is the bitmap completely clear?
|
||||
bool mi_bitmap_is_all_clear(mi_bitmap_t* bitmap);
|
||||
|
||||
// Is a sequence of n bits already set?
|
||||
// (Used to check if a memory range is already committed)
|
||||
static inline bool mi_bitmap_is_setN(mi_bitmap_t* bitmap, size_t idx, size_t n) {
|
||||
return mi_bitmap_is_xsetN(MI_BIT_SET, bitmap, idx, n);
|
||||
}
|
||||
|
||||
// Is a sequence of n bits already clear?
|
||||
static inline bool mi_bitmap_is_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n) {
|
||||
return mi_bitmap_is_xsetN(MI_BIT_CLEAR, bitmap, idx, n);
|
||||
}
|
||||
|
||||
static inline bool mi_bitmap_is_set(mi_bitmap_t* bitmap, size_t idx) {
|
||||
return mi_bitmap_is_setN(bitmap, idx, 1);
|
||||
}
|
||||
|
||||
static inline bool mi_bitmap_is_clear(mi_bitmap_t* bitmap, size_t idx) {
|
||||
return mi_bitmap_is_clearN(bitmap, idx, 1);
|
||||
}
|
||||
|
||||
// Called once a bit is cleared to see if the memory slice can be claimed.
|
||||
typedef bool (mi_claim_fun_t)(size_t slice_index, mi_arena_t* arena, bool* keep_set);
|
||||
|
||||
// Find a set bits in the bitmap, atomically clear it, and check if `claim` returns true.
|
||||
// If not claimed, continue on (potentially setting the bit again depending on `keep_set`).
|
||||
// Returns true on success, and in that case sets the index: `0 <= *pidx <= MI_BITMAP_MAX_BITS-n`.
|
||||
mi_decl_nodiscard bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx,
|
||||
mi_claim_fun_t* claim, mi_arena_t* arena );
|
||||
|
||||
|
||||
// Atomically clear a bit but only if it is set. Will block otherwise until the bit is set.
|
||||
// This is used to delay free-ing a page that it at the same time being considered to be
|
||||
// allocated from `mi_arena_try_abandoned` (and is in the `claim` function of `mi_bitmap_try_find_and_claim`).
|
||||
void mi_bitmap_clear_once_set(mi_bitmap_t* bitmap, size_t idx);
|
||||
|
||||
|
||||
// If a bit is set in the bitmap, return `true` and set `idx` to the index of the highest bit.
|
||||
// Otherwise return `false` (and `*idx` is undefined).
|
||||
// Used for unloading arena's
|
||||
bool mi_bitmap_bsr(mi_bitmap_t* bitmap, size_t* idx);
|
||||
|
||||
// Return count of all set bits in a bitmap.
|
||||
size_t mi_bitmap_popcount(mi_bitmap_t* bitmap);
|
||||
|
||||
|
||||
typedef bool (mi_forall_set_fun_t)(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg2);
|
||||
|
||||
// Visit all set bits in a bitmap (`slice_count == 1`)
|
||||
bool _mi_bitmap_forall_set(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg);
|
||||
|
||||
// Visit all set bits in a bitmap with larger ranges if possible (`slice_count >= 1`)
|
||||
// Ranges will never cross chunk boundaries though (and `slice_count <= MI_BCHUNK_BITS`)
|
||||
bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg);
|
||||
|
||||
// Visit all set bits in a bitmap with at least `rngslices` at a time (and aligned to `rngslices`).
|
||||
// This is used by purging to not break up transparent huge pages for example.
|
||||
// Ranges will never cross chunk boundaries (and `slice_count <= MI_BCHUNK_BITS`).
|
||||
bool _mi_bitmap_forall_setc_rangesn(mi_bitmap_t* bitmap, size_t rngslices, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg);
|
||||
|
||||
// Count all set bits in given range in the bitmap.
|
||||
size_t mi_bitmap_popcountN( mi_bitmap_t* bitmap, size_t idx, size_t n);
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
Binned concurrent bitmap
|
||||
Assigns a size class to each chunk such that small blocks don't cause too
|
||||
much fragmentation since we keep chunks for larger blocks separate.
|
||||
---------------------------------------------------------------------------- */
|
||||
|
||||
// mi_chunkbin_t is defined in mimalloc-stats.h
|
||||
|
||||
static inline mi_chunkbin_t mi_chunkbin_inc(mi_chunkbin_t bbin) {
|
||||
mi_assert_internal(bbin < MI_CBIN_COUNT);
|
||||
return (mi_chunkbin_t)((int)bbin + 1);
|
||||
}
|
||||
|
||||
static inline mi_chunkbin_t mi_chunkbin_dec(mi_chunkbin_t bbin) {
|
||||
mi_assert_internal(bbin > MI_CBIN_NONE);
|
||||
return (mi_chunkbin_t)((int)bbin - 1);
|
||||
}
|
||||
|
||||
static inline mi_chunkbin_t mi_chunkbin_of(size_t slice_count) {
|
||||
if (slice_count==1) return MI_CBIN_SMALL;
|
||||
if (slice_count==8) return MI_CBIN_MEDIUM;
|
||||
#if MI_ENABLE_LARGE_PAGES
|
||||
if (slice_count==MI_BFIELD_BITS) return MI_CBIN_LARGE;
|
||||
#endif
|
||||
if (slice_count > MI_BCHUNK_BITS) return MI_CBIN_HUGE;
|
||||
return MI_CBIN_OTHER;
|
||||
}
|
||||
|
||||
// An atomic "binned" bitmap for the free slices where we keep chunks reserved for particular size classes
|
||||
typedef mi_decl_bchunk_align struct mi_bbitmap_s {
|
||||
_Atomic(size_t) chunk_count; // total count of chunks (0 < N <= MI_BCHUNKMAP_BITS)
|
||||
_Atomic(size_t) chunk_max_accessed; // max chunk index that was once cleared or set
|
||||
#if (MI_BCHUNK_SIZE / MI_SIZE_SIZE) > 2
|
||||
size_t _padding[MI_BCHUNK_SIZE/MI_SIZE_SIZE - 2]; // suppress warning on msvc by aligning manually
|
||||
#endif
|
||||
mi_bchunkmap_t chunkmap;
|
||||
mi_bchunkmap_t chunkmap_bins[MI_CBIN_COUNT - 1]; // chunkmaps with bit set if the chunk is in that size class (excluding MI_CBIN_NONE)
|
||||
mi_bchunk_t chunks[MI_BITMAP_DEFAULT_CHUNK_COUNT]; // usually dynamic MI_BITMAP_MAX_CHUNK_COUNT
|
||||
} mi_bbitmap_t;
|
||||
|
||||
|
||||
static inline size_t mi_bbitmap_chunk_count(const mi_bbitmap_t* bbitmap) {
|
||||
return mi_atomic_load_relaxed(&((mi_bbitmap_t*)bbitmap)->chunk_count);
|
||||
}
|
||||
|
||||
static inline size_t mi_bbitmap_max_bits(const mi_bbitmap_t* bbitmap) {
|
||||
return (mi_bbitmap_chunk_count(bbitmap) * MI_BCHUNK_BITS);
|
||||
}
|
||||
|
||||
mi_chunkbin_t mi_bbitmap_debug_get_bin(const mi_bchunk_t* chunkmap_bins, size_t chunk_idx);
|
||||
|
||||
size_t mi_bbitmap_size(size_t bit_count, size_t* chunk_count);
|
||||
|
||||
// If a bit is clear in the bitmap, return `true` and set `idx` to the index of the highest bit that was clear.
|
||||
// Otherwise return `false` (and `*idx` is undefined).
|
||||
// Used for debug output.
|
||||
bool mi_bbitmap_bsr_inv(mi_bbitmap_t* bbitmap, size_t* idx);
|
||||
|
||||
// Initialize a bitmap to all clear; avoid a mem_zero if `already_zero` is true
|
||||
// returns the size of the bitmap.
|
||||
size_t mi_bbitmap_init(mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero);
|
||||
|
||||
// Set/clear a sequence of `n` bits in the bitmap (and can cross chunks).
|
||||
// Not atomic so only use if still local to a thread.
|
||||
void mi_bbitmap_unsafe_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n);
|
||||
|
||||
|
||||
// Set a sequence of `n` bits in the bbitmap; returns `true` if atomically transitioned from all 0's to 1's
|
||||
bool mi_bbitmap_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n);
|
||||
|
||||
|
||||
// Is a sequence of n bits already all set/cleared?
|
||||
bool mi_bbitmap_is_xsetN(mi_xset_t set, mi_bbitmap_t* bbitmap, size_t idx, size_t n);
|
||||
|
||||
// Is a sequence of n bits already set?
|
||||
// (Used to check if a memory range is already committed)
|
||||
static inline bool mi_bbitmap_is_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {
|
||||
return mi_bbitmap_is_xsetN(MI_BIT_SET, bbitmap, idx, n);
|
||||
}
|
||||
|
||||
// Is a sequence of n bits already clear?
|
||||
static inline bool mi_bbitmap_is_clearN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {
|
||||
return mi_bbitmap_is_xsetN(MI_BIT_CLEAR, bbitmap, idx, n);
|
||||
}
|
||||
|
||||
|
||||
// Try to atomically transition `n` bits from all set to all clear. Returns `true` on succes.
|
||||
// `n` cannot cross chunk boundaries, where `n <= MI_CHUNK_BITS`.
|
||||
bool mi_bbitmap_try_clearNC(mi_bbitmap_t* bbitmap, size_t idx, size_t n);
|
||||
|
||||
|
||||
// Specialized versions for common bit sequence sizes
|
||||
bool mi_bbitmap_try_find_and_clear(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx); // 1-bit
|
||||
bool mi_bbitmap_try_find_and_clear8(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx); // 8-bits
|
||||
// bool mi_bbitmap_try_find_and_clearX(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx); // MI_BFIELD_BITS
|
||||
bool mi_bbitmap_try_find_and_clearNX(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx); // < MI_BFIELD_BITS
|
||||
bool mi_bbitmap_try_find_and_clearNC(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx); // > MI_BFIELD_BITS <= MI_BCHUNK_BITS
|
||||
bool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx); // > MI_BCHUNK_BITS
|
||||
|
||||
// Find a sequence of `n` bits in the bbitmap with all bits set, and try to atomically clear all.
|
||||
// Returns true on success, and in that case sets the index: `0 <= *pidx <= MI_BITMAP_MAX_BITS-n`.
|
||||
mi_decl_nodiscard static inline bool mi_bbitmap_try_find_and_clearN(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) {
|
||||
if (n==1) return mi_bbitmap_try_find_and_clear(bbitmap, tseq, pidx); // small pages
|
||||
if (n==8) return mi_bbitmap_try_find_and_clear8(bbitmap, tseq, pidx); // medium pages
|
||||
// if (n==MI_BFIELD_BITS) return mi_bbitmap_try_find_and_clearX(bbitmap, tseq, pidx); // large pages
|
||||
if (n==0) return false;
|
||||
if (n<=MI_BFIELD_BITS) return mi_bbitmap_try_find_and_clearNX(bbitmap, tseq, n, pidx);
|
||||
if (n<=MI_BCHUNK_BITS) return mi_bbitmap_try_find_and_clearNC(bbitmap, tseq, n, pidx);
|
||||
return mi_bbitmap_try_find_and_clearN_(bbitmap, tseq, n, pidx);
|
||||
}
|
||||
|
||||
|
||||
#endif // MI_BITMAP_H
|
||||
@@ -0,0 +1,658 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#if !defined(MI_IN_ALLOC_C)
|
||||
#error "this file should be included from 'alloc.c' (so aliases can work from alloc-override)"
|
||||
// add includes help an IDE
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc/internal.h"
|
||||
#include "mimalloc/prim.h" // _mi_prim_thread_id()
|
||||
#endif
|
||||
|
||||
// forward declarations
|
||||
static void mi_check_padding(const mi_page_t* page, const mi_block_t* block);
|
||||
static bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block);
|
||||
static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool was_guarded);
|
||||
static void mi_stat_free(const mi_page_t* page, const mi_block_t* block);
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Free
|
||||
// ------------------------------------------------------
|
||||
|
||||
// regular free of a (thread local) block pointer
|
||||
// fast path written carefully to prevent spilling on the stack
|
||||
static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool was_guarded, bool track_stats, bool check_full)
|
||||
{
|
||||
MI_UNUSED(was_guarded);
|
||||
// checks
|
||||
if mi_unlikely(mi_check_is_double_free(page, block)) return;
|
||||
if (!was_guarded) { mi_check_padding(page, block); }
|
||||
if (track_stats) { mi_stat_free(page, block); }
|
||||
#if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN
|
||||
memset(block, MI_DEBUG_FREED, mi_page_block_size(page));
|
||||
#endif
|
||||
if (track_stats) { mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded)); } // faster then mi_usable_size as we already know the page and that p is unaligned
|
||||
|
||||
// actual free: push on the local free list
|
||||
mi_block_set_next(page, block, page->local_free);
|
||||
page->local_free = block;
|
||||
if mi_unlikely(--page->used == 0) {
|
||||
if (page->retire_expire==0) { // no need to re-retire retired pages (happens when we alloc/free one block repeatedly in an empty page)
|
||||
_mi_page_retire(page);
|
||||
}
|
||||
}
|
||||
else if mi_unlikely(check_full && mi_page_is_in_full(page)) {
|
||||
_mi_page_unfull(page);
|
||||
}
|
||||
}
|
||||
|
||||
// Forward declaration for multi-threaded collect
|
||||
static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept;
|
||||
|
||||
// Free a block multi-threaded
|
||||
static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded) mi_attr_noexcept
|
||||
{
|
||||
MI_UNUSED(was_guarded);
|
||||
// adjust stats (after padding check and potentially recursive `mi_free` above)
|
||||
mi_stat_free(page, block); // stat_free may access the padding
|
||||
mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded));
|
||||
|
||||
// _mi_padding_shrink(page, block, sizeof(mi_block_t));
|
||||
#if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threading
|
||||
if (!was_guarded) {
|
||||
size_t dbgsize = mi_usable_size(block);
|
||||
if (dbgsize > MI_MiB) { dbgsize = MI_MiB; }
|
||||
_mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize);
|
||||
}
|
||||
#endif
|
||||
|
||||
// push atomically on the page thread free list
|
||||
mi_thread_free_t tf_new;
|
||||
mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free);
|
||||
do {
|
||||
mi_block_set_next(page, block, mi_tf_block(tf_old));
|
||||
tf_new = mi_tf_create(block, true /* always use owned: try to claim it if the page is abandoned */);
|
||||
} while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new)); // todo: release is enough?
|
||||
|
||||
// and atomically try to collect the page if it was abandoned
|
||||
const bool is_owned_now = !mi_tf_is_owned(tf_old);
|
||||
if (is_owned_now) {
|
||||
mi_assert_internal(mi_page_is_abandoned(page));
|
||||
mi_free_try_collect_mt(page,block);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Adjust a block that was allocated aligned, to the actual start of the block in the page.
|
||||
// note: this can be called from `mi_free_generic_mt` where a non-owning thread accesses the
|
||||
// `page_start` and `block_size` fields; however these are constant and the page won't be
|
||||
// deallocated (as the block we are freeing keeps it alive) and thus safe to read concurrently.
|
||||
mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) {
|
||||
mi_assert_internal(page!=NULL && p!=NULL);
|
||||
|
||||
const size_t diff = (uint8_t*)p - mi_page_start(page);
|
||||
const size_t block_size = mi_page_block_size(page);
|
||||
const size_t adjust = (_mi_is_power_of_two(block_size) ? diff & (block_size - 1) : diff % block_size);
|
||||
return (mi_block_t*)((uintptr_t)p - adjust);
|
||||
}
|
||||
|
||||
// forward declaration for a MI_GUARDED build
|
||||
#if MI_GUARDED
|
||||
static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p); // forward declaration
|
||||
static inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) {
|
||||
if (mi_block_ptr_is_guarded(block, p)) {
|
||||
mi_block_unguard(page, block, p);
|
||||
return true;
|
||||
}
|
||||
else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
#else
|
||||
static inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) {
|
||||
MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p);
|
||||
return false;
|
||||
}
|
||||
#endif
|
||||
|
||||
static inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, void* p ) {
|
||||
mi_assert(_mi_page_ptr_unalign(page,p) == (mi_block_t*)p); // should never be an interior pointer
|
||||
#if MI_SECURE > 0
|
||||
// in secure mode we always unalign to guard against free-ing interior pointers
|
||||
return _mi_page_ptr_unalign(page,p);
|
||||
#else
|
||||
MI_UNUSED(page);
|
||||
return (mi_block_t*)p;
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
// free a local pointer (page parameter comes first for better codegen)
|
||||
static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, void* p) mi_attr_noexcept {
|
||||
mi_assert_internal(p!=NULL && page != NULL);
|
||||
mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p));
|
||||
const bool was_guarded = mi_block_check_unguard(page, block, p);
|
||||
mi_free_block_local(page, block, was_guarded, true /* track stats */, true /* check for a full page */);
|
||||
}
|
||||
|
||||
// free a pointer owned by another thread (page parameter comes first for better codegen)
|
||||
static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, void* p) mi_attr_noexcept {
|
||||
mi_assert_internal(p!=NULL && page != NULL);
|
||||
mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p));
|
||||
const bool was_guarded = mi_block_check_unguard(page, block, p);
|
||||
mi_free_block_mt(page, block, was_guarded);
|
||||
}
|
||||
|
||||
// generic free (for runtime integration)
|
||||
void mi_decl_noinline _mi_free_generic(mi_page_t* page, bool is_local, void* p) mi_attr_noexcept {
|
||||
if (is_local) mi_free_generic_local(page,p);
|
||||
else mi_free_generic_mt(page,p);
|
||||
}
|
||||
|
||||
|
||||
// Get the page belonging to a pointer
|
||||
// Does further checks in debug mode to see if this was a valid pointer.
|
||||
static inline mi_page_t* mi_validate_ptr_page(const void* p, const char* msg)
|
||||
{
|
||||
MI_UNUSED_RELEASE(msg);
|
||||
#if MI_DEBUG
|
||||
if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0 && !mi_option_is_enabled(mi_option_guarded_precise)) {
|
||||
_mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p);
|
||||
return NULL;
|
||||
}
|
||||
mi_page_t* page = _mi_safe_ptr_page(p);
|
||||
if (p != NULL && page == NULL) {
|
||||
_mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p);
|
||||
}
|
||||
return page;
|
||||
#else
|
||||
return _mi_ptr_page(p);
|
||||
#endif
|
||||
}
|
||||
|
||||
// Free a block
|
||||
// Fast path written carefully to prevent register spilling on the stack
|
||||
static mi_decl_forceinline void mi_free_ex(void* p, size_t* usable, mi_page_t* page)
|
||||
{
|
||||
if mi_unlikely(page==NULL) return; // page will be NULL if p==NULL
|
||||
mi_assert_internal(p!=NULL && page!=NULL);
|
||||
if (usable!=NULL) { *usable = mi_page_usable_block_size(page); }
|
||||
|
||||
const mi_threadid_t xtid = (_mi_prim_thread_id() ^ mi_page_xthread_id(page));
|
||||
if mi_likely(xtid == 0) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) == 0`
|
||||
// thread-local, aligned, and not a full page
|
||||
mi_block_t* const block = mi_validate_block_from_ptr(page,p);
|
||||
mi_free_block_local(page, block, false /* was guarded */, true /* track stats */, false /* no need to check if the page is full */);
|
||||
}
|
||||
else if (xtid <= MI_PAGE_FLAG_MASK) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) != 0`
|
||||
// page is local, but is full or contains (inner) aligned blocks; use generic path
|
||||
mi_free_generic_local(page, p);
|
||||
}
|
||||
// free-ing in a page owned by a theap in another thread, or an abandoned page (not belonging to a theap)
|
||||
else if ((xtid & MI_PAGE_FLAG_MASK) == 0) { // `tid != mi_page_thread_id(page) && mi_page_flags(page) == 0`
|
||||
// blocks are aligned (and not a full page); push on the thread_free list
|
||||
mi_block_t* const block = mi_validate_block_from_ptr(page,p);
|
||||
mi_free_block_mt(page,block,false /* was_guarded */);
|
||||
}
|
||||
else {
|
||||
// page is full or contains (inner) aligned blocks; use generic multi-thread path
|
||||
mi_free_generic_mt(page, p);
|
||||
}
|
||||
}
|
||||
|
||||
void mi_free(void* p) mi_attr_noexcept {
|
||||
mi_page_t* const page = mi_validate_ptr_page(p,"mi_free");
|
||||
mi_free_ex(p, NULL, page);
|
||||
}
|
||||
|
||||
void mi_ufree(void* p, size_t* usable) mi_attr_noexcept {
|
||||
mi_page_t* const page = mi_validate_ptr_page(p,"mi_ufree");
|
||||
mi_free_ex(p, usable, page);
|
||||
}
|
||||
|
||||
void mi_free_small(void* p) mi_attr_noexcept {
|
||||
// We can only call `mi_free_small` for pointers allocated with `mi_(heap_)malloc_small`.
|
||||
// If we keep page info in front of the page area for small objects, we can find the info
|
||||
// just by aligning down the pointer instead of looking it up in the page map.
|
||||
#if MI_PAGE_META_ALIGNED_FREE_SMALL
|
||||
#if MI_GUARDED
|
||||
#warning "MI_PAGE_META_ALIGNED_FREE_SMALL ignored as MI_GUARDED is defined"
|
||||
mi_free(p);
|
||||
#elif MI_ARENA_SLICE_ALIGN < MI_SMALL_PAGE_SIZE
|
||||
#warning "MI_PAGE_META_ALIGNED_FREE_SMALL ignored as the MI_ARENA_SLICE_ALIGN is less than the small page size"
|
||||
mi_free(p);
|
||||
#else
|
||||
mi_page_t* const page = (mi_page_t*)_mi_align_down_ptr(p,MI_SMALL_PAGE_SIZE);
|
||||
mi_assert(page == mi_validate_ptr_page(p,"mi_free_small"));
|
||||
mi_assert((void*)page == _mi_align_down_ptr(page->page_start,MI_SMALL_PAGE_SIZE));
|
||||
mi_assert(page->block_size <= MI_SMALL_SIZE_MAX); // note: not `MI_SMALL_MAX_OBJ_SIZE` as we need to match `mi_(heap_)malloc_small`
|
||||
mi_free_ex(p, NULL, page);
|
||||
#endif
|
||||
#else
|
||||
mi_free(p);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
// --------------------------------------------------------------------------------------------
|
||||
// `mi_free_try_collect_mt`: Potentially collect a page in a free in an abandoned page.
|
||||
// 1. if the page becomes empty, free it
|
||||
// 2. if it can be reclaimed, reclaim it in our theap
|
||||
// 3. if it went to < 7/8th used, re-abandon to be mapped (so it can be found by theaps looking for free pages)
|
||||
// --------------------------------------------------------------------------------------------
|
||||
|
||||
// Helper for mi_free_try_collect_mt: free if the page has no more used blocks (this is updated by `_mi_page_free_collect(_partly)`)
|
||||
static bool mi_abandoned_page_try_free(mi_page_t* page)
|
||||
{
|
||||
if (!mi_page_all_free(page)) return false;
|
||||
// first remove it from the abandoned pages in the arena (if mapped, this might wait for any readers to finish)
|
||||
_mi_arenas_page_unabandon(page,NULL);
|
||||
_mi_arenas_page_free(page,NULL); // we can now free the page directly
|
||||
return true;
|
||||
}
|
||||
|
||||
// Helper for mi_free_try_collect_mt: try if we can reabandon a previously abandoned mostly full page to be mapped
|
||||
static bool mi_abandoned_page_try_reabandon_to_mapped(mi_page_t* page)
|
||||
{
|
||||
// if the page is unmapped, try to reabandon so it can possibly be mapped and found for allocations
|
||||
// We only reabandon if a full page starts to have enough blocks available to prevent immediate re-abandon of a full page
|
||||
if (mi_page_is_mostly_used(page)) return false; // not too full
|
||||
if (page->memid.memkind != MI_MEM_ARENA || mi_page_is_abandoned_mapped(page)) return false; // and not already mapped (or unmappable)
|
||||
|
||||
mi_assert(!mi_page_is_full(page));
|
||||
return _mi_arenas_page_try_reabandon_to_mapped(page);
|
||||
}
|
||||
|
||||
// Release ownership of a page. This may free or reabandoned the page if other blocks are concurrently
|
||||
// freed in the meantime. Returns `true` if the page was freed.
|
||||
// By passing the captured `expected_thread_free`, we can often avoid calling `mi_page_free_collect`.
|
||||
static void mi_abandoned_page_unown_from_free(mi_page_t* page, mi_block_t* expected_thread_free) {
|
||||
mi_assert_internal(mi_page_is_owned(page));
|
||||
mi_assert_internal(mi_page_is_abandoned(page));
|
||||
mi_assert_internal(!mi_page_all_free(page));
|
||||
// try to cas atomically the original free list (`mt_free`) back with the ownership cleared.
|
||||
mi_thread_free_t tf_expect = mi_tf_create(expected_thread_free, true);
|
||||
mi_thread_free_t tf_new = mi_tf_create(expected_thread_free, false);
|
||||
while mi_unlikely(!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_expect, tf_new)) {
|
||||
mi_assert_internal(mi_tf_is_owned(tf_expect));
|
||||
// while the xthread_free list is not empty..
|
||||
while (mi_tf_block(tf_expect) != NULL) {
|
||||
// if there were concurrent updates to the thread-free list, we retry to free or reabandon to mapped (if it became !mosty_used).
|
||||
_mi_page_free_collect(page,false); // update used count
|
||||
if (mi_abandoned_page_try_free(page)) return;
|
||||
if (mi_abandoned_page_try_reabandon_to_mapped(page)) return;
|
||||
// otherwise continue un-owning
|
||||
tf_expect = mi_atomic_load_relaxed(&page->xthread_free);
|
||||
}
|
||||
// and try again to release ownership
|
||||
mi_assert_internal(mi_tf_block(tf_expect)==NULL);
|
||||
tf_new = mi_tf_create(NULL, false);
|
||||
}
|
||||
}
|
||||
|
||||
static inline bool mi_page_queue_len_is_atmost( mi_theap_t* theap, size_t block_size, long atmost) {
|
||||
if (atmost < 0) return false;
|
||||
mi_page_queue_t* const pq = mi_page_queue(theap,block_size);
|
||||
mi_assert_internal(pq!=NULL);
|
||||
return (pq->count <= (size_t)atmost);
|
||||
}
|
||||
|
||||
// Helper for mi_free_try_collect_mt: try to reclaim the page for ourselves
|
||||
static mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, long reclaim_on_free) mi_attr_noexcept
|
||||
{
|
||||
// note: reclaiming can improve benchmarks like `larson` or `rbtree-ck` a lot even in the single-threaded case,
|
||||
// since free-ing from an owned page avoids atomic operations. However, if we reclaim too eagerly in
|
||||
// a multi-threaded scenario we may start to hold on to too much memory and reduce reuse among threads.
|
||||
// If the current theap is where the page originally came from, we reclaim much more eagerly while
|
||||
// 'cross-thread' reclaiming on free is by default off (and we only 'reclaim' these by finding the abandoned
|
||||
// pages when we allocate a fresh page).
|
||||
mi_assert_internal(mi_page_is_owned(page));
|
||||
mi_assert_internal(mi_page_is_abandoned(page));
|
||||
mi_assert_internal(!mi_page_all_free(page));
|
||||
mi_assert_internal(page->block_size <= MI_MEDIUM_MAX_OBJ_SIZE);
|
||||
mi_assert_internal(reclaim_on_free >= 0);
|
||||
|
||||
// dont reclaim if we just have terminated this thread and we should
|
||||
// not reinitialize the theap for this thread. (can happen due to thread-local destructors for example -- issue #944)
|
||||
if (!_mi_thread_is_initialized()) return false;
|
||||
|
||||
// get our theap
|
||||
mi_theap_t* const theap = _mi_page_associated_theap_peek(page);
|
||||
if (theap==NULL || !theap->allow_page_reclaim) return false;
|
||||
|
||||
// todo: cache `is_in_threadpool` and `exclusive_arena` directly in the theap for performance?
|
||||
// set max_reclaim limit
|
||||
long max_reclaim = 0;
|
||||
if mi_likely(theap == page->theap) { // did this page originate from the current theap? (and thus allocated from this thread)
|
||||
// originating theap
|
||||
max_reclaim = _mi_option_get_fast(theap->tld->is_in_threadpool ? mi_option_page_cross_thread_max_reclaim : mi_option_page_max_reclaim);
|
||||
}
|
||||
else if (reclaim_on_free == 1 && // if cross-thread is allowed
|
||||
!theap->tld->is_in_threadpool && // and we are not part of a threadpool
|
||||
!mi_page_is_mostly_used(page) && // and the page is not too full
|
||||
_mi_arena_memid_is_suitable(page->memid, _mi_theap_heap(theap)->exclusive_arena)) { // and it fits our memory
|
||||
// across threads
|
||||
max_reclaim = _mi_option_get_fast(mi_option_page_cross_thread_max_reclaim);
|
||||
}
|
||||
|
||||
// are we within the reclaim limit?
|
||||
if (max_reclaim >= 0 && !mi_page_queue_len_is_atmost(theap, page->block_size, max_reclaim)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// reclaim the page into this theap
|
||||
// first remove it from the abandoned pages in the arena -- this might wait for any readers to finish
|
||||
_mi_arenas_page_unabandon(page, theap);
|
||||
_mi_theap_page_reclaim(theap, page);
|
||||
mi_theap_stat_counter_increase(theap, pages_reclaim_on_free, 1);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
// We freed a block in an abandoned page (that was not owned). Try to collect
|
||||
static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept
|
||||
{
|
||||
mi_assert_internal(mi_page_is_owned(page));
|
||||
mi_assert_internal(mi_page_is_abandoned(page));
|
||||
mi_assert_internal(mt_free != NULL);
|
||||
// we own the page now, and it is safe to collect the thread atomic free list
|
||||
if (page->block_size <= MI_SMALL_SIZE_MAX) {
|
||||
// use the `_partly` version to avoid atomic operations since we already have the `mt_free` pointing into the thread free list
|
||||
// (after this the `used` count might be too high (as some blocks may have been concurrently added to the thread free list and are yet uncounted).
|
||||
// however, if the page became completely free, the used count is guaranteed to be 0.)
|
||||
mi_assert_internal(page->reserved>=16); // below this even one freed block goes from full to no longer mostly used.
|
||||
_mi_page_free_collect_partly(page, mt_free);
|
||||
}
|
||||
else {
|
||||
// for larger blocks we use the regular collect
|
||||
_mi_page_free_collect(page,false /* no force */);
|
||||
mt_free = NULL; // expected page->xthread_free value after collection
|
||||
}
|
||||
const long reclaim_on_free = _mi_option_get_fast(mi_option_page_reclaim_on_free);
|
||||
#if MI_DEBUG > 1
|
||||
if (mi_page_is_singleton(page)) { mi_assert_internal(mi_page_all_free(page)); }
|
||||
if (mi_page_is_full(page)) { mi_assert(mi_page_is_mostly_used(page)); }
|
||||
#endif
|
||||
|
||||
// try to: 1. free it, 2. reclaim it, or 3. reabandon it to be mapped
|
||||
if (mi_abandoned_page_try_free(page)) return;
|
||||
if (page->block_size <= MI_MEDIUM_MAX_OBJ_SIZE && reclaim_on_free >= 0) { // early test for better codegen
|
||||
if (mi_abandoned_page_try_reclaim(page, reclaim_on_free)) return;
|
||||
}
|
||||
if (mi_abandoned_page_try_reabandon_to_mapped(page)) return;
|
||||
|
||||
// otherwise unown the page again
|
||||
mi_abandoned_page_unown_from_free(page, mt_free);
|
||||
}
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Usable size
|
||||
// ------------------------------------------------------
|
||||
|
||||
// Bytes available in a block
|
||||
static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* page, const void* p) mi_attr_noexcept {
|
||||
const mi_block_t* block = _mi_page_ptr_unalign(page, p);
|
||||
const bool is_guarded = mi_block_ptr_is_guarded(block,p);
|
||||
const size_t size = mi_page_usable_size_of(page, block, is_guarded);
|
||||
const ptrdiff_t adjust = (uint8_t*)p - (uint8_t*)block;
|
||||
mi_assert_internal(adjust >= 0 && (size_t)adjust <= size);
|
||||
const size_t aligned_size = (size - adjust);
|
||||
return aligned_size;
|
||||
}
|
||||
|
||||
static inline size_t _mi_usable_size(const void* p, const mi_page_t* page) mi_attr_noexcept {
|
||||
if mi_unlikely(page==NULL) return 0;
|
||||
if mi_likely(!mi_page_has_interior_pointers(page)) {
|
||||
const mi_block_t* block = (const mi_block_t*)p;
|
||||
return mi_page_usable_size_of(page, block, false /* is guarded */);
|
||||
}
|
||||
else {
|
||||
// split out to separate routine for improved code generation
|
||||
return mi_page_usable_aligned_size_of(page, p);
|
||||
}
|
||||
}
|
||||
|
||||
mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept {
|
||||
const mi_page_t* const page = mi_validate_ptr_page(p,"mi_usable_size");
|
||||
return _mi_usable_size(p,page);
|
||||
}
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Free variants
|
||||
// ------------------------------------------------------
|
||||
|
||||
void mi_free_size(void* p, size_t size) mi_attr_noexcept {
|
||||
MI_UNUSED_RELEASE(size);
|
||||
#if MI_DEBUG
|
||||
const mi_page_t* const page = mi_validate_ptr_page(p,"mi_free_size");
|
||||
const size_t available = _mi_usable_size(p,page);
|
||||
mi_assert(p == NULL || size <= available || available == 0 /* invalid pointer */ );
|
||||
#endif
|
||||
mi_free(p);
|
||||
}
|
||||
|
||||
void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept {
|
||||
MI_UNUSED_RELEASE(alignment);
|
||||
mi_assert(((uintptr_t)p % alignment) == 0);
|
||||
mi_free_size(p,size);
|
||||
}
|
||||
|
||||
void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept {
|
||||
MI_UNUSED_RELEASE(alignment);
|
||||
mi_assert(((uintptr_t)p % alignment) == 0);
|
||||
mi_free(p);
|
||||
}
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Check for double free in secure and debug mode
|
||||
// This is somewhat expensive so only enabled for secure mode 4
|
||||
// ------------------------------------------------------
|
||||
|
||||
#if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0))
|
||||
// linear check if the free list contains a specific element
|
||||
static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem) {
|
||||
while (list != NULL) {
|
||||
if (elem==list) return true;
|
||||
list = mi_block_next(page, list);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, const mi_block_t* block) {
|
||||
// The decoded value is in the same page (or NULL).
|
||||
// Walk the free lists to verify positively if it is already freed
|
||||
if (mi_list_contains(page, page->free, block) ||
|
||||
mi_list_contains(page, page->local_free, block) ||
|
||||
mi_list_contains(page, mi_page_thread_free(page), block))
|
||||
{
|
||||
_mi_error_message(EAGAIN, "double free detected of block %p with size %zu\n", block, mi_page_block_size(page));
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
#define mi_track_page(page,access) { size_t psize; void* pstart = _mi_page_start(_mi_page_segment(page),page,&psize); mi_track_mem_##access( pstart, psize); }
|
||||
|
||||
static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) {
|
||||
bool is_double_free = false;
|
||||
mi_block_t* n = mi_block_nextx(page, block, page->keys); // pretend it is freed, and get the decoded first field
|
||||
if (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer?
|
||||
(n==NULL || mi_is_in_same_page(block, n))) // quick check: in same page or NULL?
|
||||
{
|
||||
// Suspicious: decoded value a in block is in the same page (or NULL) -- maybe a double free?
|
||||
// (continue in separate function to improve code generation)
|
||||
is_double_free = mi_check_is_double_freex(page, block);
|
||||
}
|
||||
return is_double_free;
|
||||
}
|
||||
#else
|
||||
static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) {
|
||||
MI_UNUSED(page);
|
||||
MI_UNUSED(block);
|
||||
return false;
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Check for theap block overflow by setting up padding at the end of the block
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#if MI_PADDING // && !MI_TRACK_ENABLED
|
||||
static bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize) {
|
||||
*bsize = mi_page_usable_block_size(page);
|
||||
const mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize);
|
||||
mi_track_mem_defined(padding,sizeof(mi_padding_t));
|
||||
*delta = padding->delta;
|
||||
uint32_t canary = padding->canary;
|
||||
uintptr_t keys[2];
|
||||
keys[0] = page->keys[0];
|
||||
keys[1] = page->keys[1];
|
||||
bool ok = (mi_ptr_encode_canary(page,block,keys) == canary && *delta <= *bsize);
|
||||
mi_track_mem_noaccess(padding,sizeof(mi_padding_t));
|
||||
return ok;
|
||||
}
|
||||
|
||||
// Return the exact usable size of a block.
|
||||
static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) {
|
||||
if (is_guarded) {
|
||||
const size_t bsize = mi_page_block_size(page);
|
||||
return (bsize - _mi_os_page_size());
|
||||
}
|
||||
else {
|
||||
size_t bsize;
|
||||
size_t delta;
|
||||
bool ok = mi_page_decode_padding(page, block, &delta, &bsize);
|
||||
mi_assert_internal(ok); mi_assert_internal(delta <= bsize);
|
||||
return (ok ? bsize - delta : 0);
|
||||
}
|
||||
}
|
||||
|
||||
// When a non-thread-local block is freed, it becomes part of the thread delayed free
|
||||
// list that is freed later by the owning theap. If the exact usable size is too small to
|
||||
// contain the pointer for the delayed list, then shrink the padding (by decreasing delta)
|
||||
// so it will later not trigger an overflow error in `mi_free_block`.
|
||||
void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) {
|
||||
size_t bsize;
|
||||
size_t delta;
|
||||
bool ok = mi_page_decode_padding(page, block, &delta, &bsize);
|
||||
mi_assert_internal(ok);
|
||||
if (!ok || (bsize - delta) >= min_size) return; // usually already enough space
|
||||
mi_assert_internal(bsize >= min_size);
|
||||
if (bsize < min_size) return; // should never happen
|
||||
size_t new_delta = (bsize - min_size);
|
||||
mi_assert_internal(new_delta < bsize);
|
||||
mi_padding_t* padding = (mi_padding_t*)((uint8_t*)block + bsize);
|
||||
mi_track_mem_defined(padding,sizeof(mi_padding_t));
|
||||
padding->delta = (uint32_t)new_delta;
|
||||
mi_track_mem_noaccess(padding,sizeof(mi_padding_t));
|
||||
}
|
||||
#else
|
||||
static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) {
|
||||
MI_UNUSED(is_guarded); MI_UNUSED(block);
|
||||
return mi_page_usable_block_size(page);
|
||||
}
|
||||
|
||||
void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) {
|
||||
MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(min_size);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if MI_PADDING && MI_PADDING_CHECK
|
||||
|
||||
static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, size_t* size, size_t* wrong) {
|
||||
size_t bsize;
|
||||
size_t delta;
|
||||
bool ok = mi_page_decode_padding(page, block, &delta, &bsize);
|
||||
*size = *wrong = bsize;
|
||||
if (!ok) return false;
|
||||
mi_assert_internal(bsize >= delta);
|
||||
*size = bsize - delta;
|
||||
if (!mi_page_is_huge(page)) {
|
||||
uint8_t* fill = (uint8_t*)block + bsize - delta;
|
||||
const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // check at most the first N padding bytes
|
||||
mi_track_mem_defined(fill, maxpad);
|
||||
for (size_t i = 0; i < maxpad; i++) {
|
||||
if (fill[i] != MI_DEBUG_PADDING) {
|
||||
*wrong = bsize - delta + i;
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
mi_track_mem_noaccess(fill, maxpad);
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) {
|
||||
size_t size;
|
||||
size_t wrong;
|
||||
if (!mi_verify_padding(page,block,&size,&wrong)) {
|
||||
_mi_error_message(EFAULT, "buffer overflow in theap block %p of size %zu: write after %zu bytes\n", block, size, wrong );
|
||||
}
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) {
|
||||
MI_UNUSED(page);
|
||||
MI_UNUSED(block);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
// only maintain stats for smaller objects if requested
|
||||
#if (MI_STAT>0)
|
||||
static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {
|
||||
MI_UNUSED(block);
|
||||
mi_theap_t* const theap = _mi_theap_default();
|
||||
if (!mi_theap_is_initialized(theap)) return; // (for now) skip statistics if free'd after thread_done was called (usually a thread cleanup call by the OS)
|
||||
|
||||
const size_t bsize = mi_page_usable_block_size(page);
|
||||
// #if (MI_STAT>1)
|
||||
// const size_t usize = mi_page_usable_size_of(page, block);
|
||||
// mi_theap_stat_decrease(theap, malloc_requested, usize);
|
||||
// #endif
|
||||
if (bsize <= MI_LARGE_MAX_OBJ_SIZE) {
|
||||
mi_theap_stat_decrease(theap, malloc_normal, bsize);
|
||||
#if (MI_STAT > 1)
|
||||
mi_theap_stat_decrease(theap, malloc_bins[_mi_bin(bsize)], 1);
|
||||
#endif
|
||||
}
|
||||
else {
|
||||
const size_t bpsize = mi_page_block_size(page); // match stat in page.c:mi_huge_page_alloc
|
||||
mi_theap_stat_decrease(theap, malloc_huge, bpsize);
|
||||
}
|
||||
}
|
||||
#else
|
||||
void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {
|
||||
MI_UNUSED(page); MI_UNUSED(block);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
// Remove guard page when building with MI_GUARDED
|
||||
#if MI_GUARDED
|
||||
static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p) {
|
||||
MI_UNUSED(p);
|
||||
mi_assert_internal(mi_block_ptr_is_guarded(block, p));
|
||||
mi_assert_internal(mi_page_has_interior_pointers(page));
|
||||
mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t));
|
||||
mi_assert_internal(block->next == MI_BLOCK_TAG_GUARDED);
|
||||
|
||||
const size_t bsize = mi_page_block_size(page);
|
||||
const size_t psize = _mi_os_page_size();
|
||||
mi_assert_internal(bsize > psize);
|
||||
mi_assert_internal(!page->memid.is_pinned);
|
||||
void* gpage = (uint8_t*)block + bsize - psize;
|
||||
mi_assert_internal(_mi_is_aligned(gpage, psize));
|
||||
_mi_os_unprotect(gpage, psize);
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,273 @@
|
||||
/*----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc/internal.h"
|
||||
#include "mimalloc/prim.h" // _mi_theap_default
|
||||
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Heap's
|
||||
----------------------------------------------------------- */
|
||||
|
||||
mi_theap_t* mi_heap_theap(mi_heap_t* heap) {
|
||||
return _mi_heap_theap(heap);
|
||||
}
|
||||
|
||||
void mi_heap_set_numa_affinity(mi_heap_t* heap, int numa_node) {
|
||||
if (heap==NULL) { heap = mi_heap_main(); }
|
||||
heap->numa_node = (numa_node < 0 ? -1 : numa_node % _mi_os_numa_node_count());
|
||||
}
|
||||
|
||||
void mi_heap_stats_merge_to_subproc(mi_heap_t* heap) {
|
||||
if (heap==NULL) { heap = mi_heap_main(); }
|
||||
_mi_stats_merge_into(&heap->subproc->stats, &heap->stats);
|
||||
}
|
||||
|
||||
void mi_heap_stats_merge_to_main(mi_heap_t* heap) {
|
||||
if (heap==NULL) return;
|
||||
_mi_stats_merge_into(&mi_heap_main()->stats, &heap->stats);
|
||||
}
|
||||
|
||||
static mi_decl_noinline mi_theap_t* mi_heap_init_theap(const mi_heap_t* const_heap)
|
||||
{
|
||||
mi_heap_t* heap = (mi_heap_t*)const_heap;
|
||||
mi_assert_internal(heap!=NULL);
|
||||
|
||||
if (_mi_is_heap_main(heap)) {
|
||||
// this can be called if the (main) thread is not yet initialized (as no allocation happened)
|
||||
// but `theap_main_init_get()` will call `mi_thread_init()`
|
||||
mi_theap_t* const theap = _mi_theap_main_safe();
|
||||
mi_assert_internal(theap!=NULL && _mi_is_heap_main(_mi_theap_heap(theap)));
|
||||
return theap;
|
||||
}
|
||||
|
||||
// otherwise initialize the theap for this heap
|
||||
// get the thread local
|
||||
mi_assert_internal(heap->theap != 0);
|
||||
if (heap->theap==0) { // paranoia
|
||||
_mi_error_message(EFAULT, "no thread-local reserved for heap (%p)\n", heap);
|
||||
return NULL;
|
||||
}
|
||||
mi_theap_t* theap = (mi_theap_t*)_mi_thread_local_get(heap->theap);
|
||||
|
||||
// create a fresh theap?
|
||||
if (theap==NULL) {
|
||||
// set first an invalid value to ensure the thread local storage is allocated
|
||||
if (!_mi_thread_local_set(heap->theap, (mi_theap_t*)1)) {
|
||||
_mi_error_message(EFAULT, "unable to allocate memory for thread local storage\n");
|
||||
return NULL;
|
||||
}
|
||||
// then allocate the theap
|
||||
theap = _mi_theap_create(heap, _mi_theap_default_safe()->tld);
|
||||
_mi_thread_local_set(heap->theap, theap); // Cannot fail now as it was set before. Always set so the local is valid or NULL (and not 1)
|
||||
if (theap==NULL) {
|
||||
_mi_error_message(EFAULT, "unable to allocate memory for a thread local heap\n");
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
return theap;
|
||||
}
|
||||
|
||||
|
||||
// get the theap for a heap without initializing (and return NULL in that case)
|
||||
mi_theap_t* _mi_heap_theap_get_peek(const mi_heap_t* heap) {
|
||||
if (heap==NULL || _mi_is_heap_main(heap)) {
|
||||
return _mi_theap_main_safe();
|
||||
}
|
||||
else {
|
||||
return (mi_theap_t*)_mi_thread_local_get(heap->theap);
|
||||
}
|
||||
}
|
||||
|
||||
// get (and possibly create) the theap belonging to a heap
|
||||
mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap)
|
||||
{
|
||||
mi_theap_t* theap = _mi_heap_theap_peek(heap);
|
||||
if mi_unlikely(theap==NULL) {
|
||||
theap = mi_heap_init_theap(heap);
|
||||
if (theap==NULL) { return (mi_theap_t*)&_mi_theap_empty_wrong; } // this will return NULL from page.c:_mi_malloc_generic
|
||||
}
|
||||
_mi_theap_cached_set(theap);
|
||||
return theap;
|
||||
}
|
||||
|
||||
|
||||
mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t exclusive_arena_id) {
|
||||
// always allocate heap data in the (subprocess) main heap
|
||||
mi_heap_t* const heap_main = mi_heap_main();
|
||||
// todo: allocate heap data in the exclusive arena ?
|
||||
mi_heap_t* const heap = (mi_heap_t*)mi_heap_zalloc( heap_main, sizeof(mi_heap_t) );
|
||||
if (heap==NULL) return NULL;
|
||||
|
||||
// reserve a thread local slot for this heap (see also issue #1230)
|
||||
const mi_thread_local_t theap_slot = _mi_thread_local_create();
|
||||
if (theap_slot == 0) {
|
||||
_mi_error_message(EFAULT, "unable to dynamically create a thread local for a heap\n");
|
||||
mi_free(heap);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// init fields
|
||||
heap->theap = theap_slot;
|
||||
heap->subproc = heap_main->subproc;
|
||||
heap->heap_seq = mi_atomic_increment_relaxed(&heap_main->subproc->heap_total_count);
|
||||
heap->exclusive_arena = _mi_arena_from_id(exclusive_arena_id);
|
||||
heap->numa_node = -1; // no initial affinity
|
||||
mi_stats_header_init(&heap->stats);
|
||||
mi_lock_init(&heap->theaps_lock);
|
||||
mi_lock_init(&heap->os_abandoned_pages_lock);
|
||||
mi_lock_init(&heap->arena_pages_lock);
|
||||
|
||||
// push onto the subproc heaps
|
||||
mi_lock(&heap->subproc->heaps_lock) {
|
||||
mi_heap_t* head = heap->subproc->heaps;
|
||||
heap->prev = NULL;
|
||||
heap->next = head;
|
||||
if (head!=NULL) { head->prev = heap; }
|
||||
heap->subproc->heaps = heap;
|
||||
}
|
||||
mi_atomic_increment_relaxed(&heap_main->subproc->heap_count);
|
||||
mi_subproc_stat_increase(heap_main->subproc, heaps, 1);
|
||||
return heap;
|
||||
}
|
||||
|
||||
mi_heap_t* mi_heap_new(void) {
|
||||
return mi_heap_new_in_arena(0);
|
||||
}
|
||||
|
||||
// free all theaps belonging to this heap (without deleting their pages as we do this arena wise for efficiency)
|
||||
static void mi_heap_free_theaps(mi_heap_t* heap) {
|
||||
// This can run concurrently with a thread that terminates (see `init.c:mi_thread_theaps_done`),
|
||||
// and we need to ensure we free theaps atomically.
|
||||
// We do this in a loop where we release the theaps_lock at every potential re-iteration to unblock
|
||||
// potential concurrent thread termination which tries to remove the theap from our theaps list.
|
||||
bool all_freed;
|
||||
do {
|
||||
all_freed = true;
|
||||
mi_theap_t* theap = NULL;
|
||||
mi_lock(&heap->theaps_lock) {
|
||||
theap = heap->theaps;
|
||||
while(theap != NULL) {
|
||||
mi_theap_t* next = theap->hnext;
|
||||
if (!_mi_theap_free(theap, false /* dont re-acquire the heap->theaps_lock */, true /* acquire the tld->theaps_lock though */ )) {
|
||||
all_freed = false;
|
||||
}
|
||||
theap = next;
|
||||
}
|
||||
}
|
||||
if (!all_freed) {
|
||||
mi_heap_stat_counter_increase(heap,heaps_delete_wait,1);
|
||||
_mi_prim_thread_yield();
|
||||
}
|
||||
else {
|
||||
mi_assert_internal(heap->theaps==NULL);
|
||||
}
|
||||
}
|
||||
while(!all_freed);
|
||||
}
|
||||
|
||||
// free the heap resources (assuming the pages are already moved/destroyed, and all theaps have been freed)
|
||||
static void mi_heap_free(mi_heap_t* heap) {
|
||||
mi_assert_internal(heap!=NULL && !_mi_is_heap_main(heap));
|
||||
|
||||
// free all arena pages infos
|
||||
mi_lock(&heap->arena_pages_lock) {
|
||||
for (size_t i = 0; i < MI_MAX_ARENAS; i++) {
|
||||
mi_arena_pages_t* arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]);
|
||||
if (arena_pages!=NULL) {
|
||||
mi_atomic_store_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i], NULL);
|
||||
mi_free(arena_pages);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// remove the heap from the subproc
|
||||
mi_heap_stats_merge_to_main(heap);
|
||||
mi_atomic_decrement_relaxed(&heap->subproc->heap_count);
|
||||
mi_subproc_stat_decrease(heap->subproc, heaps, 1);
|
||||
mi_lock(&heap->subproc->heaps_lock) {
|
||||
if (heap->next!=NULL) { heap->next->prev = heap->prev; }
|
||||
if (heap->prev!=NULL) { heap->prev->next = heap->next; }
|
||||
else { heap->subproc->heaps = heap->next; }
|
||||
}
|
||||
|
||||
_mi_thread_local_free(heap->theap);
|
||||
mi_lock_done(&heap->theaps_lock);
|
||||
mi_lock_done(&heap->os_abandoned_pages_lock);
|
||||
mi_lock_done(&heap->arena_pages_lock);
|
||||
mi_free(heap);
|
||||
}
|
||||
|
||||
void mi_heap_delete(mi_heap_t* heap) {
|
||||
if (heap==NULL) return;
|
||||
if (_mi_is_heap_main(heap)) {
|
||||
_mi_warning_message("cannot delete the main heap\n");
|
||||
return;
|
||||
}
|
||||
mi_heap_free_theaps(heap);
|
||||
_mi_heap_move_pages(heap, mi_heap_main());
|
||||
mi_heap_free(heap);
|
||||
}
|
||||
|
||||
void _mi_heap_force_destroy(mi_heap_t* heap) {
|
||||
if (heap==NULL) return;
|
||||
mi_heap_free_theaps(heap);
|
||||
_mi_heap_destroy_pages(heap);
|
||||
if (!_mi_is_heap_main(heap)) { mi_heap_free(heap); } // todo: release locks of the main heap?
|
||||
}
|
||||
|
||||
void mi_heap_destroy(mi_heap_t* heap) {
|
||||
if (heap==NULL) return;
|
||||
if (_mi_is_heap_main(heap)) {
|
||||
_mi_warning_message("cannot destroy the main heap\n");
|
||||
return;
|
||||
}
|
||||
_mi_heap_force_destroy(heap);
|
||||
}
|
||||
|
||||
mi_heap_t* mi_heap_of(const void* p) {
|
||||
mi_page_t* page = _mi_safe_ptr_page(p);
|
||||
if (page==NULL) return NULL;
|
||||
return mi_page_heap(page);
|
||||
}
|
||||
|
||||
bool mi_any_heap_contains(const void* p) {
|
||||
return (mi_heap_of(p)!=NULL);
|
||||
}
|
||||
|
||||
bool mi_heap_contains(const mi_heap_t* heap, const void* p) {
|
||||
if (heap==NULL) { heap = mi_heap_main(); }
|
||||
return (heap==mi_heap_of(p));
|
||||
}
|
||||
|
||||
// deprecated
|
||||
bool mi_check_owned(const void* p) {
|
||||
return mi_any_heap_contains(p);
|
||||
}
|
||||
|
||||
// unsafe heap utilization function for DragonFly (see issue #1258)
|
||||
// If the page of pointer `p` belongs to `heap` (or `heap==NULL`) and has less than `perc_threshold` used blocks in its used area return `true`.
|
||||
// This function is unsafe in general as it assumes we are the only thread accessing the page of `p`.
|
||||
bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc_threshold) mi_attr_noexcept {
|
||||
if (p==NULL) return false;
|
||||
const mi_page_t* const page = _mi_safe_ptr_page(p); // Get the page containing this pointer
|
||||
if (page==NULL || page->used==page->capacity || page->capacity < page->reserved) return false;
|
||||
// If the page is the head of the queue, it is currently being used for
|
||||
// allocations; we skip it to avoid immediate thrashing.
|
||||
if (page->prev == NULL) return false;
|
||||
|
||||
// match heap?
|
||||
const mi_heap_t* const page_heap = mi_page_heap(page);
|
||||
if (page_heap==NULL) return false;
|
||||
if (heap!=NULL && page_heap!=heap) return false;
|
||||
|
||||
// check utilization
|
||||
if (page->capacity==0) return false;
|
||||
if (perc_threshold>=100) return true;
|
||||
return (perc_threshold >= ((100UL*page->used) / page->capacity));
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,477 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2024, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
// --------------------------------------------------------
|
||||
// This module defines various std libc functions to reduce
|
||||
// the dependency on libc, and also prevent errors caused
|
||||
// by some libc implementations when called before `main`
|
||||
// executes (due to malloc redirection)
|
||||
// --------------------------------------------------------
|
||||
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc/internal.h"
|
||||
#include "mimalloc/prim.h" // mi_prim_getenv
|
||||
|
||||
char _mi_toupper(char c) {
|
||||
if (c >= 'a' && c <= 'z') return (c - 'a' + 'A');
|
||||
else return c;
|
||||
}
|
||||
|
||||
int _mi_strnicmp(const char* s, const char* t, size_t n) {
|
||||
mi_assert_internal(s!=NULL && t!=NULL);
|
||||
if (n == 0) return 0;
|
||||
for (; *s != 0 && *t != 0 && n > 0; s++, t++, n--) {
|
||||
if (_mi_toupper(*s) != _mi_toupper(*t)) break;
|
||||
}
|
||||
return (n == 0 ? 0 : *s - *t);
|
||||
}
|
||||
|
||||
bool _mi_streq(const char* s, const char* t) {
|
||||
if (s==NULL && t==NULL) return true;
|
||||
if (s==NULL || t==NULL) return false;
|
||||
for (; *s != 0 && *t != 0; s++, t++) {
|
||||
if (*s != *t) break;
|
||||
}
|
||||
return (*s == *t);
|
||||
}
|
||||
|
||||
void _mi_strlcpy(char* dest, const char* src, size_t dest_size) {
|
||||
if (dest==NULL || src==NULL || dest_size == 0) return;
|
||||
// copy until end of src, or when dest is (almost) full
|
||||
while (*src != 0 && dest_size > 1) {
|
||||
*dest++ = *src++;
|
||||
dest_size--;
|
||||
}
|
||||
// always zero terminate
|
||||
*dest = 0;
|
||||
}
|
||||
|
||||
void _mi_strlcat(char* dest, const char* src, size_t dest_size) {
|
||||
if (dest==NULL || src==NULL || dest_size == 0) return;
|
||||
// find end of string in the dest buffer
|
||||
while (*dest != 0 && dest_size > 1) {
|
||||
dest++;
|
||||
dest_size--;
|
||||
}
|
||||
// and catenate
|
||||
_mi_strlcpy(dest, src, dest_size);
|
||||
}
|
||||
|
||||
size_t _mi_strnlen(const char* s, size_t max_len) {
|
||||
if (s==NULL) return 0;
|
||||
size_t len = 0;
|
||||
while(s[len] != 0 && len < max_len) { len++; }
|
||||
return len;
|
||||
}
|
||||
|
||||
size_t _mi_strlen(const char* s) {
|
||||
return _mi_strnlen(s,PTRDIFF_MAX);
|
||||
}
|
||||
|
||||
char* _mi_strnstr(char* s, size_t max_len, const char* pat) {
|
||||
if (s==NULL) return NULL;
|
||||
if (pat==NULL) return s;
|
||||
const size_t m = _mi_strnlen(s, max_len);
|
||||
const size_t n = _mi_strlen(pat);
|
||||
for (size_t start = 0; start + n <= m; start++) {
|
||||
size_t i = 0;
|
||||
while (i<n && pat[i]==s[start+i]) {
|
||||
i++;
|
||||
}
|
||||
if (i==n) return &s[start];
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#ifdef MI_NO_GETENV
|
||||
bool _mi_getenv(const char* name, char* result, size_t result_size) {
|
||||
MI_UNUSED(name);
|
||||
MI_UNUSED(result);
|
||||
MI_UNUSED(result_size);
|
||||
return false;
|
||||
}
|
||||
#else
|
||||
bool _mi_getenv(const char* name, char* result, size_t result_size) {
|
||||
if (name==NULL || result == NULL || result_size < 64) return false;
|
||||
return _mi_prim_getenv(name,result,result_size);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
// --------------------------------------------------------
|
||||
// Define our own primitives for doing an action once
|
||||
// --------------------------------------------------------
|
||||
|
||||
// Returns `true` only on the first invocation, signifying we can execute an action once.
|
||||
// If it returns `true`, the caller should call `_mi_atomic_once_release` after performing the action.
|
||||
// Other threads (than the initial thread that entered) will block until `_mi_atomic_once_release` has been called.
|
||||
bool _mi_atomic_once_enter(mi_atomic_once_t* once) {
|
||||
const uintptr_t once_tid = mi_atomic_load_acquire(&once->tid);
|
||||
if mi_likely(once_tid == 1) {
|
||||
return false; // already executed
|
||||
}
|
||||
const mi_threadid_t current_tid = _mi_thread_id();
|
||||
if (once_tid == current_tid) {
|
||||
return false; // recursive invocation; we need this for process_init for example
|
||||
}
|
||||
|
||||
mi_lock_acquire(&once->lock);
|
||||
uintptr_t expected = 0;
|
||||
if (mi_atomic_cas_strong_acq_rel(&once->tid, &expected, current_tid)) { // could use atomic_load/store as well
|
||||
return true; // should execute and release
|
||||
}
|
||||
else {
|
||||
mi_lock_release(&once->lock);
|
||||
return false; // already another thread entered and released
|
||||
}
|
||||
}
|
||||
|
||||
void _mi_atomic_once_release(mi_atomic_once_t* once) {
|
||||
if (mi_atomic_load_acquire(&once->tid)>1) { // paranoia
|
||||
mi_atomic_store_release(&once->tid,1); // done executing
|
||||
mi_lock_release(&once->lock);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// --------------------------------------------------------
|
||||
// Define our own limited `_mi_vsnprintf` and `_mi_snprintf`
|
||||
// This is mostly to avoid calling these when libc is not yet
|
||||
// initialized (and to reduce dependencies)
|
||||
//
|
||||
// format: d i, p x u, s
|
||||
// prec: z l ll L
|
||||
// width: 10
|
||||
// align-left: -
|
||||
// fill: 0
|
||||
// plus: +
|
||||
// --------------------------------------------------------
|
||||
|
||||
static void mi_outc(char c, char** out, char* end) {
|
||||
char* p = *out;
|
||||
if (p >= end) return;
|
||||
*p = c;
|
||||
*out = p + 1;
|
||||
}
|
||||
|
||||
static void mi_outs(const char* s, char** out, char* end) {
|
||||
if (s == NULL) return;
|
||||
char* p = *out;
|
||||
while (*s != 0 && p < end) {
|
||||
*p++ = *s++;
|
||||
}
|
||||
*out = p;
|
||||
}
|
||||
|
||||
static void mi_out_fill(char fill, size_t len, char** out, char* end) {
|
||||
char* p = *out;
|
||||
for (size_t i = 0; i < len && p < end; i++) {
|
||||
*p++ = fill;
|
||||
}
|
||||
*out = p;
|
||||
}
|
||||
|
||||
static void mi_out_alignright(char fill, char* start, size_t len, size_t extra, char* end) {
|
||||
if (len == 0 || extra == 0) return;
|
||||
if (start + len + extra >= end) return;
|
||||
// move `len` characters to the right (in reverse since it can overlap)
|
||||
for (size_t i = 1; i <= len; i++) {
|
||||
start[len + extra - i] = start[len - i];
|
||||
}
|
||||
// and fill the start
|
||||
for (size_t i = 0; i < extra; i++) {
|
||||
start[i] = fill;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static void mi_out_num(uintmax_t x, size_t base, char prefix, char** out, char* end)
|
||||
{
|
||||
if (x == 0 || base == 0 || base > 16) {
|
||||
if (prefix != 0) { mi_outc(prefix, out, end); }
|
||||
mi_outc('0',out,end);
|
||||
}
|
||||
else {
|
||||
// output digits in reverse
|
||||
char* start = *out;
|
||||
while (x > 0) {
|
||||
char digit = (char)(x % base);
|
||||
mi_outc((digit <= 9 ? '0' + digit : 'A' + digit - 10),out,end);
|
||||
x = x / base;
|
||||
}
|
||||
if (prefix != 0) {
|
||||
mi_outc(prefix, out, end);
|
||||
}
|
||||
size_t len = *out - start;
|
||||
// and reverse in-place
|
||||
for (size_t i = 0; i < (len / 2); i++) {
|
||||
char c = start[len - i - 1];
|
||||
start[len - i - 1] = start[i];
|
||||
start[i] = c;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#define MI_NEXTC() c = *in; if (c==0) break; in++;
|
||||
|
||||
int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) {
|
||||
if (buf == NULL || bufsize == 0 || fmt == NULL) return 0;
|
||||
buf[bufsize - 1] = 0;
|
||||
char* const end = buf + (bufsize - 1);
|
||||
const char* in = fmt;
|
||||
char* out = buf;
|
||||
while (true) {
|
||||
if (out >= end) break;
|
||||
char c;
|
||||
MI_NEXTC();
|
||||
if (c != '%') {
|
||||
if (c == '\\') {
|
||||
MI_NEXTC();
|
||||
switch (c) {
|
||||
case 'e': mi_outc('\x1B', &out, end); break;
|
||||
case 't': mi_outc('\t', &out, end); break;
|
||||
case 'n': mi_outc('\n', &out, end); break;
|
||||
case 'r': mi_outc('\r', &out, end); break;
|
||||
case '\\': mi_outc('\\', &out, end); break;
|
||||
default: /* ignore */ break;
|
||||
}
|
||||
}
|
||||
else if ((c >= ' ' && c <= '~') || c=='\n' || c=='\r' || c=='\t' || c=='\x1b') { // output visible ascii or standard control only
|
||||
mi_outc(c, &out, end);
|
||||
}
|
||||
}
|
||||
else {
|
||||
MI_NEXTC();
|
||||
char fill = ' ';
|
||||
size_t width = 0;
|
||||
char numtype = 'd';
|
||||
char numplus = 0;
|
||||
bool alignright = true;
|
||||
if (c == '+' || c == ' ') { numplus = c; MI_NEXTC(); }
|
||||
if (c == '-') { alignright = false; MI_NEXTC(); }
|
||||
if (c == '0') { fill = '0'; MI_NEXTC(); }
|
||||
if (c >= '1' && c <= '9') {
|
||||
width = (c - '0'); MI_NEXTC();
|
||||
while (c >= '0' && c <= '9') {
|
||||
width = (10 * width) + (c - '0'); MI_NEXTC();
|
||||
}
|
||||
if (c == 0) break; // extra check due to while
|
||||
}
|
||||
if (c == 'z' || c == 't' || c == 'L') { numtype = c; MI_NEXTC(); }
|
||||
else if (c == 'l') {
|
||||
numtype = c; MI_NEXTC();
|
||||
if (c == 'l') { numtype = 'L'; MI_NEXTC(); }
|
||||
}
|
||||
|
||||
char* start = out;
|
||||
if (c == '%') {
|
||||
mi_outc('%', &out, end);
|
||||
}
|
||||
else if (c == 's') {
|
||||
// string
|
||||
const char* s = va_arg(args, const char*);
|
||||
mi_outs(s, &out, end);
|
||||
}
|
||||
else if (c == 'p' || c == 'x' || c == 'u') {
|
||||
// unsigned
|
||||
uintmax_t x = 0;
|
||||
if (c == 'x' || c == 'u') {
|
||||
if (numtype == 'z') x = va_arg(args, size_t);
|
||||
else if (numtype == 't') x = va_arg(args, uintptr_t); // unsigned ptrdiff_t
|
||||
else if (numtype == 'L') x = va_arg(args, unsigned long long);
|
||||
else if (numtype == 'l') x = va_arg(args, unsigned long);
|
||||
else x = va_arg(args, unsigned int);
|
||||
}
|
||||
else if (c == 'p') {
|
||||
void* const p = va_arg(args, void*);
|
||||
x = (uintptr_t)p;
|
||||
mi_outs("0x", &out, end);
|
||||
start = out;
|
||||
width = (width >= 2 ? width - 2 : 0);
|
||||
}
|
||||
if (width == 0 && (c == 'x' || c == 'p')) {
|
||||
if (c == 'p') { width = 2 * (x <= UINT32_MAX ? 4 : ((x >> 16) <= UINT32_MAX ? 6 : sizeof(void*))); }
|
||||
if (width == 0) { width = 2; }
|
||||
fill = '0';
|
||||
}
|
||||
mi_out_num(x, (c == 'x' || c == 'p' ? 16 : 10), numplus, &out, end);
|
||||
}
|
||||
else if (c == 'i' || c == 'd') {
|
||||
// signed
|
||||
intmax_t x = 0;
|
||||
if (numtype == 'z') x = va_arg(args, intptr_t );
|
||||
else if (numtype == 't') x = va_arg(args, ptrdiff_t);
|
||||
else if (numtype == 'L') x = va_arg(args, long long);
|
||||
else if (numtype == 'l') x = va_arg(args, long);
|
||||
else x = va_arg(args, int);
|
||||
char pre = 0;
|
||||
if (x < 0) {
|
||||
pre = '-';
|
||||
if (x > INTMAX_MIN) { x = -x; }
|
||||
}
|
||||
else if (numplus != 0) {
|
||||
pre = numplus;
|
||||
}
|
||||
mi_out_num((uintmax_t)x, 10, pre, &out, end);
|
||||
}
|
||||
else if (c >= ' ' && c <= '~') {
|
||||
// unknown format
|
||||
mi_outc('%', &out, end);
|
||||
mi_outc(c, &out, end);
|
||||
}
|
||||
|
||||
// fill & align
|
||||
mi_assert_internal(out <= end);
|
||||
mi_assert_internal(out >= start);
|
||||
const size_t len = out - start;
|
||||
if (len < width) {
|
||||
mi_out_fill(fill, width - len, &out, end);
|
||||
if (alignright && out <= end) {
|
||||
mi_out_alignright(fill, start, len, width - len, end);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
mi_assert_internal(out <= end);
|
||||
*out = 0;
|
||||
return (int)(out - buf);
|
||||
}
|
||||
|
||||
int _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...) {
|
||||
va_list args;
|
||||
va_start(args, fmt);
|
||||
const int written = _mi_vsnprintf(buf, buflen, fmt, args);
|
||||
va_end(args);
|
||||
return written;
|
||||
}
|
||||
|
||||
|
||||
|
||||
// --------------------------------------------------------
|
||||
// generic trailing and leading zero count, and popcount
|
||||
// --------------------------------------------------------
|
||||
|
||||
#if !MI_HAS_FAST_BITSCAN
|
||||
|
||||
static size_t mi_ctz_generic32(uint32_t x) {
|
||||
// de Bruijn multiplication, see <http://keithandkatie.com/keith/papers/debruijn.html>
|
||||
static const uint8_t debruijn[32] = {
|
||||
0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
|
||||
31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9
|
||||
};
|
||||
if (x==0) return 32;
|
||||
return debruijn[(uint32_t)((x & -(int32_t)x) * (uint32_t)(0x077CB531U)) >> 27];
|
||||
}
|
||||
|
||||
static size_t mi_clz_generic32(uint32_t x) {
|
||||
// de Bruijn multiplication, see <http://keithandkatie.com/keith/papers/debruijn.html>
|
||||
static const uint8_t debruijn[32] = {
|
||||
31, 22, 30, 21, 18, 10, 29, 2, 20, 17, 15, 13, 9, 6, 28, 1,
|
||||
23, 19, 11, 3, 16, 14, 7, 24, 12, 4, 8, 25, 5, 26, 27, 0
|
||||
};
|
||||
if (x==0) return 32;
|
||||
x |= x >> 1;
|
||||
x |= x >> 2;
|
||||
x |= x >> 4;
|
||||
x |= x >> 8;
|
||||
x |= x >> 16;
|
||||
return debruijn[(uint32_t)(x * (uint32_t)(0x07C4ACDDU)) >> 27];
|
||||
}
|
||||
|
||||
size_t _mi_ctz_generic(size_t x) {
|
||||
if (x==0) return MI_SIZE_BITS;
|
||||
#if (MI_SIZE_BITS <= 32)
|
||||
return mi_ctz_generic32((uint32_t)x);
|
||||
#else
|
||||
const uint32_t lo = (uint32_t)x;
|
||||
if (lo != 0) {
|
||||
return mi_ctz_generic32(lo);
|
||||
}
|
||||
else {
|
||||
return (32 + mi_ctz_generic32((uint32_t)(x>>32)));
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
size_t _mi_clz_generic(size_t x) {
|
||||
if (x==0) return MI_SIZE_BITS;
|
||||
#if (MI_SIZE_BITS <= 32)
|
||||
return mi_clz_generic32((uint32_t)x);
|
||||
#else
|
||||
const uint32_t hi = (uint32_t)(x>>32);
|
||||
if (hi != 0) {
|
||||
return mi_clz_generic32(hi);
|
||||
}
|
||||
else {
|
||||
return 32 + mi_clz_generic32((uint32_t)x);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif // bit scan
|
||||
|
||||
|
||||
#if MI_SIZE_SIZE == 4
|
||||
#define mi_mask_even_bits32 (0x55555555)
|
||||
#define mi_mask_even_pairs32 (0x33333333)
|
||||
#define mi_mask_even_nibbles32 (0x0F0F0F0F)
|
||||
|
||||
// sum of all the bytes in `x` if it is guaranteed that the sum < 256!
|
||||
static size_t mi_byte_sum32(uint32_t x) {
|
||||
// perform `x * 0x01010101`: the highest byte contains the sum of all bytes.
|
||||
x += (x << 8);
|
||||
x += (x << 16);
|
||||
return (size_t)(x >> 24);
|
||||
}
|
||||
|
||||
static size_t mi_popcount_generic32(uint32_t x) {
|
||||
// first count each 2-bit group `a`, where: a==0b00 -> 00, a==0b01 -> 01, a==0b10 -> 01, a==0b11 -> 10
|
||||
// in other words, `a - (a>>1)`; to do this in parallel, we need to mask to prevent spilling a bit pair
|
||||
// into the lower bit-pair:
|
||||
x = x - ((x >> 1) & mi_mask_even_bits32);
|
||||
// add the 2-bit pair results
|
||||
x = (x & mi_mask_even_pairs32) + ((x >> 2) & mi_mask_even_pairs32);
|
||||
// add the 4-bit nibble results
|
||||
x = (x + (x >> 4)) & mi_mask_even_nibbles32;
|
||||
// each byte now has a count of its bits, we can sum them now:
|
||||
return mi_byte_sum32(x);
|
||||
}
|
||||
|
||||
mi_decl_noinline size_t _mi_popcount_generic(size_t x) {
|
||||
if (x<=1) return x;
|
||||
if (~x==0) return MI_SIZE_BITS;
|
||||
return mi_popcount_generic32(x);
|
||||
}
|
||||
|
||||
#else
|
||||
#define mi_mask_even_bits64 (0x5555555555555555)
|
||||
#define mi_mask_even_pairs64 (0x3333333333333333)
|
||||
#define mi_mask_even_nibbles64 (0x0F0F0F0F0F0F0F0F)
|
||||
|
||||
// sum of all the bytes in `x` if it is guaranteed that the sum < 256!
|
||||
static size_t mi_byte_sum64(uint64_t x) {
|
||||
x += (x << 8);
|
||||
x += (x << 16);
|
||||
x += (x << 32);
|
||||
return (size_t)(x >> 56);
|
||||
}
|
||||
|
||||
static size_t mi_popcount_generic64(uint64_t x) {
|
||||
x = x - ((x >> 1) & mi_mask_even_bits64);
|
||||
x = (x & mi_mask_even_pairs64) + ((x >> 2) & mi_mask_even_pairs64);
|
||||
x = (x + (x >> 4)) & mi_mask_even_nibbles64;
|
||||
return mi_byte_sum64(x);
|
||||
}
|
||||
|
||||
mi_decl_noinline size_t _mi_popcount_generic(size_t x) {
|
||||
if (x<=1) return x;
|
||||
if (~x==0) return MI_SIZE_BITS;
|
||||
return mi_popcount_generic64(x);
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,704 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc/internal.h"
|
||||
#include "mimalloc/atomic.h"
|
||||
#include "mimalloc/prim.h" // mi_prim_out_stderr
|
||||
|
||||
#include <stdio.h> // stdin/stdout
|
||||
#include <stdlib.h> // abort
|
||||
|
||||
static long mi_max_error_count = 16; // stop outputting errors after this (use < 0 for no limit)
|
||||
static long mi_max_warning_count = 16; // stop outputting warnings after this (use < 0 for no limit)
|
||||
|
||||
static void mi_add_stderr_output(void);
|
||||
|
||||
int mi_version(void) mi_attr_noexcept {
|
||||
return MI_MALLOC_VERSION;
|
||||
}
|
||||
|
||||
|
||||
// --------------------------------------------------------
|
||||
// Options
|
||||
// These can be accessed by multiple threads and may be
|
||||
// concurrently initialized, but an initializing data race
|
||||
// is ok since they resolve to the same value.
|
||||
// --------------------------------------------------------
|
||||
|
||||
|
||||
#define MI_OPTION(opt) mi_option_##opt, #opt, NULL
|
||||
#define MI_OPTION_LEGACY(opt,legacy) mi_option_##opt, #opt, #legacy
|
||||
|
||||
// Some options can be set at build time for statically linked libraries
|
||||
// (use `-DMI_EXTRA_CPPDEFS="opt1=val1;opt2=val2"`)
|
||||
//
|
||||
// This is useful if we cannot pass them as environment variables
|
||||
// (and setting them programmatically would be too late)
|
||||
|
||||
#ifndef MI_DEFAULT_VERBOSE
|
||||
#define MI_DEFAULT_VERBOSE 0
|
||||
#endif
|
||||
|
||||
#ifndef MI_DEFAULT_ARENA_EAGER_COMMIT
|
||||
#define MI_DEFAULT_ARENA_EAGER_COMMIT 2
|
||||
#endif
|
||||
|
||||
// in KiB
|
||||
#ifndef MI_DEFAULT_ARENA_RESERVE
|
||||
#if (MI_INTPTR_SIZE>4)
|
||||
#define MI_DEFAULT_ARENA_RESERVE 1024L*1024L
|
||||
#else
|
||||
#define MI_DEFAULT_ARENA_RESERVE 128L*1024L
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifndef MI_DEFAULT_ARENA_MAX_OBJECT_SIZE
|
||||
#define MI_DEFAULT_ARENA_MAX_OBJECT_SIZE ((MI_SIZE_BITS * MI_ARENA_MAX_CHUNK_OBJ_SIZE)/MI_KiB) /* 2 GiB (or 256 MiB on 32-bit), larger than this is alloc'd by the OS */
|
||||
#endif
|
||||
|
||||
#ifndef MI_DEFAULT_DISALLOW_ARENA_ALLOC
|
||||
#define MI_DEFAULT_DISALLOW_ARENA_ALLOC 0
|
||||
#endif
|
||||
|
||||
#ifndef MI_DEFAULT_ALLOW_LARGE_OS_PAGES
|
||||
#define MI_DEFAULT_ALLOW_LARGE_OS_PAGES 0
|
||||
#endif
|
||||
|
||||
#ifndef MI_DEFAULT_RESERVE_HUGE_OS_PAGES
|
||||
#define MI_DEFAULT_RESERVE_HUGE_OS_PAGES 0
|
||||
#endif
|
||||
|
||||
#ifndef MI_DEFAULT_RESERVE_OS_MEMORY
|
||||
#define MI_DEFAULT_RESERVE_OS_MEMORY 0
|
||||
#endif
|
||||
|
||||
#ifndef MI_DEFAULT_GUARDED_SAMPLE_RATE
|
||||
#if MI_GUARDED && !MI_DEBUG
|
||||
#define MI_DEFAULT_GUARDED_SAMPLE_RATE 4000
|
||||
#else
|
||||
#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifndef MI_DEFAULT_PAGEMAP_COMMIT
|
||||
#if defined(__APPLE__) // when overloading malloc, we still get mixed pointers sometimes on macOS; this avoids a bad access
|
||||
#define MI_DEFAULT_PAGEMAP_COMMIT 1
|
||||
#else
|
||||
#define MI_DEFAULT_PAGEMAP_COMMIT 0
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifndef MI_DEFAULT_PAGE_MAX_RECLAIM
|
||||
#define MI_DEFAULT_PAGE_MAX_RECLAIM (-1) // unlimited
|
||||
#endif
|
||||
|
||||
#ifndef MI_DEFAULT_PAGE_CROSS_THREAD_MAX_RECLAIM
|
||||
#define MI_DEFAULT_PAGE_CROSS_THREAD_MAX_RECLAIM 32
|
||||
#endif
|
||||
|
||||
#ifndef MI_DEFAULT_ALLOW_THP
|
||||
#if defined(__ANDROID__)
|
||||
#define MI_DEFAULT_ALLOW_THP 0
|
||||
#else
|
||||
#define MI_DEFAULT_ALLOW_THP 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// Static options
|
||||
static mi_option_desc_t mi_options[_mi_option_last] =
|
||||
{
|
||||
// stable options
|
||||
#if MI_DEBUG || defined(MI_SHOW_ERRORS)
|
||||
{ 1, MI_OPTION_UNINIT, MI_OPTION(show_errors) },
|
||||
#else
|
||||
{ 0, MI_OPTION_UNINIT, MI_OPTION(show_errors) },
|
||||
#endif
|
||||
{ 0, MI_OPTION_UNINIT, MI_OPTION(show_stats) },
|
||||
{ MI_DEFAULT_VERBOSE, MI_OPTION_UNINIT, MI_OPTION(verbose) },
|
||||
|
||||
// some of the following options are experimental and not all combinations are allowed.
|
||||
{ 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_eager_commit) },
|
||||
{ MI_DEFAULT_ARENA_EAGER_COMMIT,
|
||||
MI_OPTION_UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, // eager commit arena's? 2 is used to enable this only on an OS that has overcommit (i.e. linux)
|
||||
{ 1, MI_OPTION_UNINIT, MI_OPTION_LEGACY(purge_decommits,reset_decommits) }, // purge decommits memory (instead of reset) (note: on linux this uses MADV_DONTNEED for decommit)
|
||||
{ MI_DEFAULT_ALLOW_LARGE_OS_PAGES,
|
||||
MI_OPTION_UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
|
||||
{ MI_DEFAULT_RESERVE_HUGE_OS_PAGES,
|
||||
MI_OPTION_UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages
|
||||
{-1, MI_OPTION_UNINIT, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N
|
||||
{ MI_DEFAULT_RESERVE_OS_MEMORY,
|
||||
MI_OPTION_UNINIT, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`)
|
||||
{ 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_segment_cache) }, // cache N segments per thread
|
||||
{ 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_page_reset) }, // reset page memory on free
|
||||
{ 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_abandoned_page_purge) },
|
||||
{ 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_segment_reset) }, // reset segment memory on free (needs eager commit)
|
||||
{ 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_eager_commit_delay) },
|
||||
{ 1000,MI_OPTION_UNINIT, MI_OPTION_LEGACY(purge_delay,reset_delay) }, // purge delay in milli-seconds
|
||||
{ 0, MI_OPTION_UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes.
|
||||
{ 0, MI_OPTION_UNINIT, MI_OPTION_LEGACY(disallow_os_alloc,limit_os_alloc) }, // 1 = do not use OS memory for allocation (but only reserved arenas)
|
||||
{ 100, MI_OPTION_UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose
|
||||
{ 32, MI_OPTION_UNINIT, MI_OPTION(max_errors) }, // maximum errors that are output
|
||||
{ 32, MI_OPTION_UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output
|
||||
{ 10, MI_OPTION_UNINIT, MI_OPTION(deprecated_max_segment_reclaim)}, // max. percentage of the abandoned segments to be reclaimed per try.
|
||||
{ 0, MI_OPTION_UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees!
|
||||
{ MI_DEFAULT_ARENA_RESERVE, MI_OPTION_UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`)
|
||||
{ 1, MI_OPTION_UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's
|
||||
{ 1, MI_OPTION_UNINIT, MI_OPTION_LEGACY(deprecated_purge_extend_delay, decommit_extend_delay) },
|
||||
{ MI_DEFAULT_DISALLOW_ARENA_ALLOC, MI_OPTION_UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's)
|
||||
{ 400, MI_OPTION_UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries.
|
||||
#if defined(MI_VISIT_ABANDONED)
|
||||
{ 1, MI_OPTION_INITIALIZED, MI_OPTION(visit_abandoned) }, // allow visiting theap blocks in abandoned segments; requires taking locks during reclaim.
|
||||
#else
|
||||
{ 0, MI_OPTION_UNINIT, MI_OPTION(visit_abandoned) },
|
||||
#endif
|
||||
{ 0, MI_OPTION_UNINIT, MI_OPTION(guarded_min) }, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects
|
||||
{ MI_GiB, MI_OPTION_UNINIT, MI_OPTION(guarded_max) }, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects
|
||||
{ 0, MI_OPTION_UNINIT, MI_OPTION(guarded_precise) }, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0)
|
||||
{ MI_DEFAULT_GUARDED_SAMPLE_RATE,
|
||||
MI_OPTION_UNINIT, MI_OPTION(guarded_sample_rate)}, // 1 out of N allocations in the min/max range will be guarded (=4000)
|
||||
{ 0, MI_OPTION_UNINIT, MI_OPTION(guarded_sample_seed)},
|
||||
{ 10000, MI_OPTION_UNINIT, MI_OPTION(generic_collect) }, // collect theaps every N (=10000) generic allocation calls
|
||||
{ 0, MI_OPTION_UNINIT, MI_OPTION_LEGACY(page_reclaim_on_free, abandoned_reclaim_on_free) },// reclaim abandoned (small) pages on a free: -1 = disable completely, 0 = only reclaim into the originating theap, 1 = reclaim on free across theaps
|
||||
{ 2, MI_OPTION_UNINIT, MI_OPTION(page_full_retain) }, // number of (small) pages to retain in the free page queues
|
||||
{ 4, MI_OPTION_UNINIT, MI_OPTION(page_max_candidates) }, // max search to find a best page candidate
|
||||
{ 0, MI_OPTION_UNINIT, MI_OPTION(max_vabits) }, // max virtual address space bits
|
||||
{ MI_DEFAULT_PAGEMAP_COMMIT,
|
||||
MI_OPTION_UNINIT, MI_OPTION(pagemap_commit) }, // commit the full pagemap upfront?
|
||||
{ 0, MI_OPTION_UNINIT, MI_OPTION(page_commit_on_demand) }, // commit pages on-demand (2 disables this only on overcommit systems (like Linux))
|
||||
{ MI_DEFAULT_PAGE_MAX_RECLAIM,
|
||||
MI_OPTION_UNINIT, MI_OPTION(page_max_reclaim) }, // don't reclaim (small) pages of the same originating theap if we already own N pages in that size class
|
||||
{ MI_DEFAULT_PAGE_CROSS_THREAD_MAX_RECLAIM,
|
||||
MI_OPTION_UNINIT, MI_OPTION(page_cross_thread_max_reclaim) }, // don't reclaim (small) pages across threads if we already own N pages in that size class
|
||||
{ MI_DEFAULT_ALLOW_THP,
|
||||
MI_OPTION_UNINIT, MI_OPTION(allow_thp) }, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process.
|
||||
{ 0, MI_OPTION_UNINIT, MI_OPTION(minimal_purge_size) }, // set minimal purge size (in KiB) (=0). Using 0 resolves to either 64 (or 2048 if `mi_option_allow_thp==2`).
|
||||
{ MI_DEFAULT_ARENA_MAX_OBJECT_SIZE,
|
||||
MI_OPTION_UNINIT, MI_OPTION(arena_max_object_size) }, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit).
|
||||
{ 0, MI_OPTION_UNINIT, MI_OPTION(arena_is_numa_local) }, // associate local numa node with an initial arena allocation
|
||||
};
|
||||
|
||||
static void mi_option_init(mi_option_desc_t* desc);
|
||||
|
||||
static bool mi_option_has_size_in_kib(mi_option_t option) {
|
||||
return (option == mi_option_reserve_os_memory || option == mi_option_arena_reserve ||
|
||||
option == mi_option_minimal_purge_size || option == mi_option_arena_max_object_size);
|
||||
}
|
||||
|
||||
void _mi_options_init(void) {
|
||||
// called on process load
|
||||
for(int i = 0; i < _mi_option_last; i++ ) {
|
||||
mi_option_t option = (mi_option_t)i;
|
||||
long l = mi_option_get(option); MI_UNUSED(l); // initialize
|
||||
}
|
||||
mi_max_error_count = mi_option_get(mi_option_max_errors);
|
||||
mi_max_warning_count = mi_option_get(mi_option_max_warnings);
|
||||
#if MI_GUARDED
|
||||
if (mi_option_get(mi_option_guarded_sample_rate) > 0) {
|
||||
if (mi_option_is_enabled(mi_option_allow_large_os_pages)) {
|
||||
mi_option_disable(mi_option_allow_large_os_pages);
|
||||
_mi_warning_message("option 'allow_large_os_pages' is disabled to allow for guarded objects\n");
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
// called at actual process load, it should be safe to print now
|
||||
void _mi_options_post_init(void) {
|
||||
mi_add_stderr_output(); // now it safe to use stderr for output
|
||||
if (mi_option_is_enabled(mi_option_verbose)) { mi_options_print(); }
|
||||
}
|
||||
|
||||
#define mi_stringifyx(str) #str // and stringify
|
||||
#define mi_stringify(str) mi_stringifyx(str) // expand
|
||||
|
||||
mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept
|
||||
{
|
||||
// show version
|
||||
const int vermajor = MI_MALLOC_VERSION/10000;
|
||||
const int verminor = (MI_MALLOC_VERSION%10000)/100;
|
||||
const int verpatch = (MI_MALLOC_VERSION%100);
|
||||
_mi_fprintf(out, arg, "v%i.%i.%i%s%s (built on %s, %s)\n", vermajor, verminor, verpatch,
|
||||
#if defined(MI_CMAKE_BUILD_TYPE)
|
||||
", " mi_stringify(MI_CMAKE_BUILD_TYPE)
|
||||
#else
|
||||
""
|
||||
#endif
|
||||
,
|
||||
#if defined(MI_GIT_DESCRIBE)
|
||||
", git " mi_stringify(MI_GIT_DESCRIBE)
|
||||
#else
|
||||
""
|
||||
#endif
|
||||
, __DATE__, __TIME__);
|
||||
|
||||
// show options
|
||||
for (int i = 0; i < _mi_option_last; i++) {
|
||||
mi_option_t option = (mi_option_t)i;
|
||||
long l = mi_option_get(option); MI_UNUSED(l); // possibly initialize
|
||||
mi_option_desc_t* desc = &mi_options[option];
|
||||
_mi_fprintf(out, arg, "option '%s': %ld %s\n", desc->name, desc->value, (mi_option_has_size_in_kib(option) ? "KiB" : ""));
|
||||
}
|
||||
|
||||
// show build configuration
|
||||
_mi_fprintf(out, arg, "debug level : %d\n", MI_DEBUG );
|
||||
_mi_fprintf(out, arg, "secure level: %d\n", MI_SECURE );
|
||||
_mi_fprintf(out, arg, "mem tracking: %s\n", MI_TRACK_TOOL);
|
||||
#if MI_GUARDED
|
||||
_mi_fprintf(out, arg, "guarded build: %s\n", mi_option_get(mi_option_guarded_sample_rate) != 0 ? "enabled" : "disabled");
|
||||
#endif
|
||||
#if MI_TSAN
|
||||
_mi_fprintf(out, arg, "thread santizer enabled\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
mi_decl_export void mi_options_print(void) mi_attr_noexcept {
|
||||
mi_options_print_out(NULL, NULL);
|
||||
}
|
||||
|
||||
long _mi_option_get_fast(mi_option_t option) {
|
||||
mi_assert(option >= 0 && option < _mi_option_last);
|
||||
mi_option_desc_t* desc = &mi_options[option];
|
||||
mi_assert(desc->option == option); // index should match the option
|
||||
//mi_assert(desc->init != MI_OPTION_UNINIT);
|
||||
return desc->value;
|
||||
}
|
||||
|
||||
|
||||
mi_decl_nodiscard long mi_option_get(mi_option_t option) {
|
||||
mi_assert(option >= 0 && option < _mi_option_last);
|
||||
if (option < 0 || option >= _mi_option_last) return 0;
|
||||
mi_option_desc_t* desc = &mi_options[option];
|
||||
mi_assert(desc->option == option); // index should match the option
|
||||
if mi_unlikely(desc->init == MI_OPTION_UNINIT) {
|
||||
mi_option_init(desc);
|
||||
}
|
||||
return desc->value;
|
||||
}
|
||||
|
||||
mi_decl_nodiscard long mi_option_get_clamp(mi_option_t option, long min, long max) {
|
||||
long x = mi_option_get(option);
|
||||
return (x < min ? min : (x > max ? max : x));
|
||||
}
|
||||
|
||||
mi_decl_nodiscard size_t mi_option_get_size(mi_option_t option) {
|
||||
const long x = mi_option_get(option);
|
||||
size_t size = (x < 0 ? 0 : (size_t)x);
|
||||
if (mi_option_has_size_in_kib(option)) {
|
||||
if (mi_mul_overflow(size, MI_KiB, &size)) {
|
||||
size = MI_MAX_ALLOC_SIZE;
|
||||
}
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
void mi_option_set(mi_option_t option, long value) {
|
||||
mi_assert(option >= 0 && option < _mi_option_last);
|
||||
if (option < 0 || option >= _mi_option_last) return;
|
||||
mi_option_desc_t* desc = &mi_options[option];
|
||||
mi_assert(desc->option == option); // index should match the option
|
||||
desc->value = value;
|
||||
desc->init = MI_OPTION_INITIALIZED;
|
||||
// ensure min/max range; be careful to not recurse.
|
||||
if (desc->option == mi_option_guarded_min && _mi_option_get_fast(mi_option_guarded_max) < value) {
|
||||
mi_option_set(mi_option_guarded_max, value);
|
||||
}
|
||||
else if (desc->option == mi_option_guarded_max && _mi_option_get_fast(mi_option_guarded_min) > value) {
|
||||
mi_option_set(mi_option_guarded_min, value);
|
||||
}
|
||||
}
|
||||
|
||||
void mi_option_set_default(mi_option_t option, long value) {
|
||||
mi_assert(option >= 0 && option < _mi_option_last);
|
||||
if (option < 0 || option >= _mi_option_last) return;
|
||||
mi_option_desc_t* desc = &mi_options[option];
|
||||
if (desc->init != MI_OPTION_INITIALIZED) {
|
||||
desc->value = value;
|
||||
}
|
||||
}
|
||||
|
||||
mi_decl_nodiscard bool mi_option_is_enabled(mi_option_t option) {
|
||||
return (mi_option_get(option) != 0);
|
||||
}
|
||||
|
||||
void mi_option_set_enabled(mi_option_t option, bool enable) {
|
||||
mi_option_set(option, (enable ? 1 : 0));
|
||||
}
|
||||
|
||||
void mi_option_set_enabled_default(mi_option_t option, bool enable) {
|
||||
mi_option_set_default(option, (enable ? 1 : 0));
|
||||
}
|
||||
|
||||
void mi_option_enable(mi_option_t option) {
|
||||
mi_option_set_enabled(option,true);
|
||||
}
|
||||
|
||||
void mi_option_disable(mi_option_t option) {
|
||||
mi_option_set_enabled(option,false);
|
||||
}
|
||||
|
||||
static void mi_cdecl mi_out_stderr(const char* msg, void* arg) {
|
||||
MI_UNUSED(arg);
|
||||
if (msg != NULL && msg[0] != 0) {
|
||||
_mi_prim_out_stderr(msg);
|
||||
}
|
||||
}
|
||||
|
||||
// Since an output function can be registered earliest in the `main`
|
||||
// function we also buffer output that happens earlier. When
|
||||
// an output function is registered it is called immediately with
|
||||
// the output up to that point.
|
||||
#ifndef MI_MAX_DELAY_OUTPUT
|
||||
#define MI_MAX_DELAY_OUTPUT ((size_t)(16*1024))
|
||||
#endif
|
||||
static char out_buf[MI_MAX_DELAY_OUTPUT+1];
|
||||
static _Atomic(size_t) out_len;
|
||||
static mi_lock_t out_buf_lock = MI_LOCK_INITIALIZER;
|
||||
|
||||
static void mi_cdecl mi_out_buf(const char* msg, void* arg) {
|
||||
MI_UNUSED(arg);
|
||||
if (msg==NULL) return;
|
||||
if (mi_atomic_load_acquire(&out_len)>=MI_MAX_DELAY_OUTPUT) return;
|
||||
size_t n = _mi_strlen(msg);
|
||||
if (n==0 || n >= MI_MAX_DELAY_OUTPUT) return;
|
||||
// copy msg into the buffer
|
||||
mi_lock(&out_buf_lock) {
|
||||
const size_t start = mi_atomic_add_acq_rel(&out_len, n);
|
||||
if (start < MI_MAX_DELAY_OUTPUT) {
|
||||
// check bound
|
||||
if (start+n >= MI_MAX_DELAY_OUTPUT) {
|
||||
n = MI_MAX_DELAY_OUTPUT-start-1;
|
||||
}
|
||||
_mi_memcpy(&out_buf[start], msg, n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void mi_out_buf_flush(mi_output_fun* out, bool no_more_buf, void* arg) {
|
||||
if (out==NULL) return;
|
||||
// claim (if `no_more_buf == true`, no more output will be added after this point)
|
||||
mi_lock(&out_buf_lock) {
|
||||
size_t count = mi_atomic_add_acq_rel(&out_len, (no_more_buf ? MI_MAX_DELAY_OUTPUT : 1));
|
||||
// and output the current contents
|
||||
if (count>MI_MAX_DELAY_OUTPUT) count = MI_MAX_DELAY_OUTPUT;
|
||||
out_buf[count] = 0;
|
||||
out(out_buf,arg);
|
||||
if (!no_more_buf) {
|
||||
out_buf[count] = '\n'; // if continue with the buffer, insert a newline
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Once this module is loaded, switch to this routine
|
||||
// which outputs to stderr and the delayed output buffer.
|
||||
static void mi_cdecl mi_out_buf_stderr(const char* msg, void* arg) {
|
||||
mi_out_stderr(msg,arg);
|
||||
mi_out_buf(msg,arg);
|
||||
}
|
||||
|
||||
|
||||
|
||||
// --------------------------------------------------------
|
||||
// Default output handler
|
||||
// --------------------------------------------------------
|
||||
|
||||
// The program should only install a single output handler from a single thread
|
||||
// since otherwise the argument and output function may not match.
|
||||
static _Atomic(void*) mi_out_default; // = // is `mi_output_fun*` (but some platforms don't support atomic function pointers)
|
||||
static _Atomic(void*) mi_out_arg; // = NULL
|
||||
|
||||
static mi_output_fun* mi_out_get_default(void** parg) {
|
||||
mi_output_fun* const out = (mi_output_fun*)mi_atomic_load_ptr_acquire(void,&mi_out_default);
|
||||
if (parg != NULL) { *parg = mi_atomic_load_ptr_acquire(void,&mi_out_arg); }
|
||||
return (out == NULL ? &mi_out_buf : out);
|
||||
}
|
||||
|
||||
void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept {
|
||||
mi_atomic_store_ptr_release(void,&mi_out_default, (void*)(out == NULL ? &mi_out_stderr : out)); // stop using the delayed output buffer
|
||||
mi_atomic_store_ptr_release(void,&mi_out_arg, arg);
|
||||
if (out!=NULL) { mi_out_buf_flush(out,true,arg); } // output all the delayed output now
|
||||
}
|
||||
|
||||
// add stderr to the delayed output after the module is loaded
|
||||
static void mi_add_stderr_output(void) {
|
||||
mi_assert_internal(mi_out_default == NULL);
|
||||
mi_out_buf_flush(&mi_out_stderr, false, NULL); // flush current contents to stderr
|
||||
mi_atomic_store_ptr_release(void,&mi_out_default,(void*)&mi_out_buf_stderr); // and add stderr to the delayed output
|
||||
mi_atomic_store_ptr_release(void,&mi_out_arg,NULL);
|
||||
}
|
||||
|
||||
// --------------------------------------------------------
|
||||
// Messages, all end up calling `_mi_fputs`.
|
||||
// --------------------------------------------------------
|
||||
static _Atomic(size_t) error_count; // = 0; // when >= max_error_count stop emitting errors
|
||||
static _Atomic(size_t) warning_count; // = 0; // when >= max_warning_count stop emitting warnings
|
||||
|
||||
// When overriding malloc, we may recurse into mi_vfprintf if an allocation
|
||||
// inside the C runtime causes another message.
|
||||
// In some cases (like on macOS) the loader already allocates which
|
||||
// calls into mimalloc; if we then access thread locals (like `recurse`)
|
||||
// this may crash as the access may call _tlv_bootstrap that tries to
|
||||
// (recursively) invoke malloc again to allocate space for the thread local
|
||||
// variables on demand. This is why we use a _mi_preloading test on such
|
||||
// platforms. However, C code generator may move the initial thread local address
|
||||
// load before the `if` and we therefore split it out in a separate function.
|
||||
static mi_decl_thread bool recurse = false;
|
||||
|
||||
static mi_decl_noinline bool mi_recurse_enter_prim(void) {
|
||||
if (recurse) return false;
|
||||
recurse = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
static mi_decl_noinline void mi_recurse_exit_prim(void) {
|
||||
recurse = false;
|
||||
}
|
||||
|
||||
static bool mi_recurse_enter(void) {
|
||||
#if defined(__APPLE__) || defined(__ANDROID__) || defined(MI_TLS_RECURSE_GUARD)
|
||||
if (_mi_preloading()) return false;
|
||||
#endif
|
||||
return mi_recurse_enter_prim();
|
||||
}
|
||||
|
||||
static void mi_recurse_exit(void) {
|
||||
#if defined(__APPLE__) || defined(__ANDROID__) || defined(MI_TLS_RECURSE_GUARD)
|
||||
if (_mi_preloading()) return;
|
||||
#endif
|
||||
mi_recurse_exit_prim();
|
||||
}
|
||||
|
||||
void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message) {
|
||||
if (out==NULL || (void*)out==(void*)stdout || (void*)out==(void*)stderr) { // TODO: use mi_out_stderr for stderr?
|
||||
if (!mi_recurse_enter()) return;
|
||||
out = mi_out_get_default(&arg);
|
||||
if (prefix != NULL) out(prefix, arg);
|
||||
out(message, arg);
|
||||
mi_recurse_exit();
|
||||
}
|
||||
else {
|
||||
if (prefix != NULL) out(prefix, arg);
|
||||
out(message, arg);
|
||||
}
|
||||
}
|
||||
|
||||
// Define our own limited `fprintf` that avoids memory allocation.
|
||||
// We do this using `_mi_vsnprintf` with a limited buffer.
|
||||
static void mi_vfprintf( mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args ) {
|
||||
char buf[992];
|
||||
if (fmt==NULL) return;
|
||||
if (!mi_recurse_enter()) return;
|
||||
_mi_vsnprintf(buf, sizeof(buf)-1, fmt, args);
|
||||
mi_recurse_exit();
|
||||
_mi_fputs(out,arg,prefix,buf);
|
||||
}
|
||||
|
||||
void _mi_fprintf( mi_output_fun* out, void* arg, const char* fmt, ... ) {
|
||||
va_list args;
|
||||
va_start(args,fmt);
|
||||
mi_vfprintf(out,arg,NULL,fmt,args);
|
||||
va_end(args);
|
||||
}
|
||||
|
||||
static void mi_vfprintf_thread(mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args) {
|
||||
if (prefix != NULL && _mi_strnlen(prefix,33) <= 32 && !_mi_is_main_thread()) {
|
||||
char tprefix[64];
|
||||
_mi_snprintf(tprefix, sizeof(tprefix), "%sthread 0x%tx: ", prefix, (uintptr_t)_mi_thread_id());
|
||||
mi_vfprintf(out, arg, tprefix, fmt, args);
|
||||
}
|
||||
else {
|
||||
mi_vfprintf(out, arg, prefix, fmt, args);
|
||||
}
|
||||
}
|
||||
|
||||
void _mi_raw_message(const char* fmt, ...) {
|
||||
va_list args;
|
||||
va_start(args, fmt);
|
||||
mi_vfprintf(NULL, NULL, NULL, fmt, args);
|
||||
va_end(args);
|
||||
}
|
||||
|
||||
void _mi_message(const char* fmt, ...) {
|
||||
va_list args;
|
||||
va_start(args, fmt);
|
||||
mi_vfprintf_thread(NULL, NULL, "mimalloc: ", fmt, args);
|
||||
va_end(args);
|
||||
}
|
||||
|
||||
void _mi_trace_message(const char* fmt, ...) {
|
||||
if (mi_option_get(mi_option_verbose) <= 1) return; // only with verbose level 2 or higher
|
||||
va_list args;
|
||||
va_start(args, fmt);
|
||||
mi_vfprintf_thread(NULL, NULL, "mimalloc: ", fmt, args);
|
||||
va_end(args);
|
||||
}
|
||||
|
||||
void _mi_verbose_message(const char* fmt, ...) {
|
||||
if (!mi_option_is_enabled(mi_option_verbose)) return;
|
||||
va_list args;
|
||||
va_start(args,fmt);
|
||||
mi_vfprintf(NULL, NULL, "mimalloc: ", fmt, args);
|
||||
va_end(args);
|
||||
}
|
||||
|
||||
static void mi_show_error_message(const char* fmt, va_list args) {
|
||||
if (!mi_option_is_enabled(mi_option_verbose)) {
|
||||
if (!mi_option_is_enabled(mi_option_show_errors)) return;
|
||||
if (mi_max_error_count >= 0 && (long)mi_atomic_increment_acq_rel(&error_count) > mi_max_error_count) return;
|
||||
}
|
||||
mi_vfprintf_thread(NULL, NULL, "mimalloc: error: ", fmt, args);
|
||||
}
|
||||
|
||||
void _mi_warning_message(const char* fmt, ...) {
|
||||
if (!mi_option_is_enabled(mi_option_verbose)) {
|
||||
if (!mi_option_is_enabled(mi_option_show_errors)) return;
|
||||
if (mi_max_warning_count >= 0 && (long)mi_atomic_increment_acq_rel(&warning_count) > mi_max_warning_count) return;
|
||||
}
|
||||
va_list args;
|
||||
va_start(args,fmt);
|
||||
mi_vfprintf_thread(NULL, NULL, "mimalloc: warning: ", fmt, args);
|
||||
va_end(args);
|
||||
}
|
||||
|
||||
|
||||
#if MI_DEBUG
|
||||
mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const char* fname, unsigned line, const char* func ) mi_attr_noexcept {
|
||||
_mi_fprintf(NULL, NULL, "mimalloc: assertion failed: at \"%s\":%u, %s\n assertion: \"%s\"\n", fname, line, (func==NULL?"":func), assertion);
|
||||
abort();
|
||||
}
|
||||
#endif
|
||||
|
||||
// --------------------------------------------------------
|
||||
// Errors
|
||||
// --------------------------------------------------------
|
||||
|
||||
static mi_error_fun* volatile mi_error_handler; // = NULL
|
||||
static _Atomic(void*) mi_error_arg; // = NULL
|
||||
|
||||
static void mi_error_default(int err) {
|
||||
MI_UNUSED(err);
|
||||
#if (MI_DEBUG>0)
|
||||
if (err==EFAULT) {
|
||||
#ifdef _MSC_VER
|
||||
__debugbreak();
|
||||
#endif
|
||||
abort();
|
||||
}
|
||||
#endif
|
||||
#if (MI_SECURE>0)
|
||||
if (err==EFAULT) { // abort on serious errors in secure mode (corrupted meta-data)
|
||||
abort();
|
||||
}
|
||||
#endif
|
||||
#if defined(MI_XMALLOC)
|
||||
if (err==ENOMEM || err==EOVERFLOW) { // abort on memory allocation fails in xmalloc mode
|
||||
abort();
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
void mi_register_error(mi_error_fun* fun, void* arg) {
|
||||
mi_error_handler = fun; // can be NULL
|
||||
mi_atomic_store_ptr_release(void,&mi_error_arg, arg);
|
||||
}
|
||||
|
||||
void _mi_error_message(int err, const char* fmt, ...) {
|
||||
// show detailed error message
|
||||
va_list args;
|
||||
va_start(args, fmt);
|
||||
mi_show_error_message(fmt, args);
|
||||
va_end(args);
|
||||
// and call the error handler which may abort (or return normally)
|
||||
if (mi_error_handler != NULL) {
|
||||
mi_error_handler(err, mi_atomic_load_ptr_acquire(void,&mi_error_arg));
|
||||
}
|
||||
else {
|
||||
mi_error_default(err);
|
||||
}
|
||||
}
|
||||
|
||||
// --------------------------------------------------------
|
||||
// Initialize options by checking the environment
|
||||
// --------------------------------------------------------
|
||||
|
||||
// TODO: implement ourselves to reduce dependencies on the C runtime
|
||||
#include <stdlib.h> // strtol
|
||||
#include <string.h> // strstr
|
||||
|
||||
|
||||
static void mi_option_init(mi_option_desc_t* desc) {
|
||||
// Read option value from the environment
|
||||
char s[64 + 1];
|
||||
char buf[64+1];
|
||||
_mi_strlcpy(buf, "mimalloc_", sizeof(buf));
|
||||
_mi_strlcat(buf, desc->name, sizeof(buf));
|
||||
bool found = _mi_getenv(buf, s, sizeof(s));
|
||||
if (!found && desc->legacy_name != NULL) {
|
||||
_mi_strlcpy(buf, "mimalloc_", sizeof(buf));
|
||||
_mi_strlcat(buf, desc->legacy_name, sizeof(buf));
|
||||
found = _mi_getenv(buf, s, sizeof(s));
|
||||
if (found) {
|
||||
_mi_warning_message("environment option \"mimalloc_%s\" is deprecated -- use \"mimalloc_%s\" instead.\n", desc->legacy_name, desc->name);
|
||||
}
|
||||
}
|
||||
|
||||
if (found) {
|
||||
size_t len = _mi_strnlen(s, sizeof(buf) - 1);
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
buf[i] = _mi_toupper(s[i]);
|
||||
}
|
||||
buf[len] = 0;
|
||||
if (buf[0] == 0 || _mi_streq(buf,"1") || _mi_streq(buf,"TRUE") || _mi_streq(buf,"YES") || _mi_streq(buf,"ON")) {
|
||||
desc->value = 1;
|
||||
desc->init = MI_OPTION_INITIALIZED;
|
||||
}
|
||||
else if (_mi_streq(buf,"0") || _mi_streq(buf,"FALSE") || _mi_streq(buf,"NO") || _mi_streq(buf,"OFF")) {
|
||||
desc->value = 0;
|
||||
desc->init = MI_OPTION_INITIALIZED;
|
||||
}
|
||||
else {
|
||||
char* end = buf;
|
||||
long value = strtol(buf, &end, 10);
|
||||
if (mi_option_has_size_in_kib(desc->option)) {
|
||||
// this option is interpreted in KiB to prevent overflow of `long` for large allocations
|
||||
// (long is 32-bit on 64-bit windows, which allows for 4TiB max.)
|
||||
size_t size = (value < 0 ? 0 : (size_t)value);
|
||||
bool overflow = false;
|
||||
if (*end == 'K') { end++; }
|
||||
else if (*end == 'M') { overflow = mi_mul_overflow(size,MI_KiB,&size); end++; }
|
||||
else if (*end == 'G') { overflow = mi_mul_overflow(size,MI_MiB,&size); end++; }
|
||||
else if (*end == 'T') { overflow = mi_mul_overflow(size,MI_GiB,&size); end++; }
|
||||
else { size = (size + MI_KiB - 1) / MI_KiB; }
|
||||
if (end[0] == 'I' && end[1] == 'B') { end += 2; } // KiB, MiB, GiB, TiB
|
||||
else if (*end == 'B') { end++; } // Kb, Mb, Gb, Tb
|
||||
if (overflow || size > (MI_MAX_ALLOC_SIZE / MI_KiB)) { size = (MI_MAX_ALLOC_SIZE / MI_KiB); }
|
||||
value = (size > LONG_MAX ? LONG_MAX : (long)size);
|
||||
}
|
||||
if (*end == 0) {
|
||||
mi_option_set(desc->option, value);
|
||||
}
|
||||
else {
|
||||
// set `init` first to avoid recursion through _mi_warning_message on mimalloc_verbose.
|
||||
desc->init = MI_OPTION_DEFAULTED;
|
||||
if (desc->option == mi_option_verbose && desc->value == 0) {
|
||||
// if the 'mimalloc_verbose' env var has a bogus value we'd never know
|
||||
// (since the value defaults to 'off') so in that case briefly enable verbose
|
||||
desc->value = 1;
|
||||
_mi_warning_message("environment option mimalloc_%s has an invalid value.\n", desc->name);
|
||||
desc->value = 0;
|
||||
}
|
||||
else {
|
||||
_mi_warning_message("environment option mimalloc_%s has an invalid value.\n", desc->name);
|
||||
}
|
||||
}
|
||||
}
|
||||
mi_assert_internal(desc->init != MI_OPTION_UNINIT);
|
||||
}
|
||||
else if (!_mi_preloading()) {
|
||||
desc->init = MI_OPTION_DEFAULTED;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,923 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc/internal.h"
|
||||
#include "mimalloc/atomic.h"
|
||||
#include "mimalloc/prim.h"
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Initialization.
|
||||
----------------------------------------------------------- */
|
||||
#ifndef MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB
|
||||
#if MI_INTPTR_SIZE < 8
|
||||
#define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 4*MI_MiB // 4 GiB
|
||||
#else
|
||||
#define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 32*MI_MiB // 32 GiB
|
||||
#endif
|
||||
#endif
|
||||
|
||||
static mi_os_mem_config_t mi_os_mem_config = {
|
||||
4096, // page size
|
||||
0, // large page size (usually 2MiB)
|
||||
4096, // allocation granularity
|
||||
MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB,
|
||||
MI_MAX_VABITS, // in `bits.h`
|
||||
true, // has overcommit? (if true we use MAP_NORESERVE on mmap systems)
|
||||
false, // can we partially free allocated blocks? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span)
|
||||
true, // has virtual reserve? (if true we can reserve virtual address space without using commit or physical memory)
|
||||
false // has transparent huge pages? (if true we purge in (aligned) large page size chunks only to not fragment such pages)
|
||||
};
|
||||
|
||||
bool _mi_os_has_overcommit(void) {
|
||||
return mi_os_mem_config.has_overcommit;
|
||||
}
|
||||
|
||||
bool _mi_os_has_virtual_reserve(void) {
|
||||
return mi_os_mem_config.has_virtual_reserve;
|
||||
}
|
||||
|
||||
|
||||
// OS (small) page size
|
||||
size_t _mi_os_page_size(void) {
|
||||
return mi_os_mem_config.page_size;
|
||||
}
|
||||
|
||||
// if large OS pages are supported (2 or 4MiB), then return the size, otherwise return the small page size (4KiB)
|
||||
size_t _mi_os_large_page_size(void) {
|
||||
return (mi_os_mem_config.large_page_size != 0 ? mi_os_mem_config.large_page_size : _mi_os_page_size());
|
||||
}
|
||||
|
||||
// minimal purge size. Can be larger than the page size if transparent huge pages are enabled.
|
||||
size_t _mi_os_minimal_purge_size(void) {
|
||||
size_t minsize = mi_option_get_size(mi_option_minimal_purge_size);
|
||||
if (minsize != 0) {
|
||||
return _mi_align_up(minsize, _mi_os_page_size());
|
||||
}
|
||||
else if (mi_os_mem_config.has_transparent_huge_pages && mi_option_get(mi_option_allow_thp) == 2) {
|
||||
return _mi_os_large_page_size();
|
||||
}
|
||||
else {
|
||||
return _mi_os_page_size();
|
||||
}
|
||||
}
|
||||
|
||||
size_t _mi_os_guard_page_size(void) {
|
||||
const size_t gsize = _mi_os_page_size();
|
||||
mi_assert(gsize <= (MI_ARENA_SLICE_SIZE/4)); // issue #1166
|
||||
return gsize;
|
||||
}
|
||||
|
||||
size_t _mi_os_virtual_address_bits(void) {
|
||||
const size_t vbits = mi_os_mem_config.virtual_address_bits;
|
||||
mi_assert(vbits <= MI_MAX_VABITS);
|
||||
return vbits;
|
||||
}
|
||||
|
||||
bool _mi_os_canuse_large_page(size_t size, size_t alignment) {
|
||||
// if we have access, check the size and alignment requirements
|
||||
if (mi_os_mem_config.large_page_size == 0) return false;
|
||||
return ((size % mi_os_mem_config.large_page_size) == 0 && (alignment % mi_os_mem_config.large_page_size) == 0);
|
||||
}
|
||||
|
||||
// round to a good OS allocation size (bounded by max 12.5% waste)
|
||||
size_t _mi_os_good_alloc_size(size_t size) {
|
||||
size_t align_size;
|
||||
if (size < 512*MI_KiB) align_size = _mi_os_page_size();
|
||||
else if (size < 2*MI_MiB) align_size = 64*MI_KiB;
|
||||
else if (size < 8*MI_MiB) align_size = 256*MI_KiB;
|
||||
else if (size < 32*MI_MiB) align_size = 1*MI_MiB;
|
||||
else align_size = 4*MI_MiB;
|
||||
if mi_unlikely(size >= (SIZE_MAX - align_size)) return size; // possible overflow?
|
||||
return _mi_align_up(size, align_size);
|
||||
}
|
||||
|
||||
void _mi_os_init(void) {
|
||||
_mi_prim_mem_init(&mi_os_mem_config);
|
||||
}
|
||||
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Util
|
||||
-------------------------------------------------------------- */
|
||||
bool _mi_os_decommit(void* addr, size_t size);
|
||||
bool _mi_os_commit(void* addr, size_t size, bool* is_zero);
|
||||
|
||||
// On systems with enough virtual address bits, we can do efficient aligned allocation by using
|
||||
// the 2TiB to 30TiB area to allocate those. If we have at least 46 bits of virtual address
|
||||
// space (64TiB) we use this technique. (but see issue #939)
|
||||
#if (MI_INTPTR_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT) // && !defined(WIN32) && !defined(ANDROID)
|
||||
|
||||
// Return a MI_HINT_ALIGN (4MiB) aligned address that is probably available.
|
||||
// If this returns NULL, the OS will determine the address but on some OS's that may not be
|
||||
// properly aligned which can be more costly as it needs to be adjusted afterwards.
|
||||
// For a size > 16GiB this always returns NULL in order to guarantee good ASLR randomization;
|
||||
// (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses
|
||||
// in the middle of the 2TiB - 6TiB address range (see issue #372))
|
||||
|
||||
#define MI_HINT_ALIGN ((uintptr_t)4 << 20) // 4MiB alignment
|
||||
#define MI_HINT_BASE ((uintptr_t)2 << 40) // 2TiB start
|
||||
#define MI_HINT_AREA ((uintptr_t)4 << 40) // upto (2+4) 6TiB (since before win8 there is "only" 8TiB available to processes)
|
||||
#define MI_HINT_MAX ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages)
|
||||
|
||||
void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size)
|
||||
{
|
||||
static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; // = 0
|
||||
|
||||
// todo: perhaps only do alignment hints if THP is enabled?
|
||||
if (try_alignment <= mi_os_mem_config.alloc_granularity || try_alignment > MI_HINT_ALIGN) return NULL;
|
||||
if (mi_os_mem_config.virtual_address_bits < 46) return NULL; // < 64TiB virtual address space
|
||||
size = _mi_align_up(size, MI_HINT_ALIGN);
|
||||
if (size > 16*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at least 1/(MI_HINT_AREA / 1<<34)
|
||||
size += MI_HINT_ALIGN; // put in virtual gaps between hinted blocks; this splits VLA's but increases guarded areas.
|
||||
|
||||
uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size);
|
||||
if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize
|
||||
uintptr_t init = MI_HINT_BASE;
|
||||
#if (MI_SECURE>=1 || defined(NDEBUG)) // security: randomize start of aligned allocations unless in debug mode
|
||||
mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267)
|
||||
if (!mi_theap_is_initialized(theap)) return NULL; // no hint as we lack randomness at this point
|
||||
const uintptr_t r = _mi_theap_random_next(theap);
|
||||
init = init + ((MI_HINT_ALIGN * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB
|
||||
#endif
|
||||
uintptr_t expected = hint + size;
|
||||
mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init);
|
||||
hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all
|
||||
}
|
||||
mi_assert_internal(hint%MI_HINT_ALIGN == 0);
|
||||
if (hint%try_alignment != 0) return NULL;
|
||||
return (void*)hint;
|
||||
}
|
||||
#else
|
||||
void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) {
|
||||
MI_UNUSED(try_alignment); MI_UNUSED(size);
|
||||
return NULL;
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Guard page allocation
|
||||
----------------------------------------------------------- */
|
||||
|
||||
// In secure mode, return the size of a guard page, otherwise 0
|
||||
size_t _mi_os_secure_guard_page_size(void) {
|
||||
#if MI_SECURE > 0
|
||||
return _mi_os_guard_page_size();
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
// In secure mode, try to decommit an area and output a warning if this fails.
|
||||
bool _mi_os_secure_guard_page_set_at(void* addr, mi_memid_t memid) {
|
||||
if (addr == NULL) return true;
|
||||
#if MI_SECURE > 0
|
||||
bool ok = false;
|
||||
if (!memid.is_pinned) {
|
||||
mi_arena_t* const arena = mi_memid_arena(memid);
|
||||
if (arena != NULL && arena->commit_fun != NULL) {
|
||||
ok = (*(arena->commit_fun))(false /* decommit */, addr, _mi_os_secure_guard_page_size(), NULL, arena->commit_fun_arg);
|
||||
}
|
||||
else {
|
||||
ok = _mi_os_decommit(addr, _mi_os_secure_guard_page_size());
|
||||
}
|
||||
}
|
||||
if (!ok) {
|
||||
_mi_error_message(EINVAL, "secure level %d, but failed to commit guard page (at %p of size %zu)\n", MI_SECURE, addr, _mi_os_secure_guard_page_size());
|
||||
}
|
||||
return ok;
|
||||
#else
|
||||
MI_UNUSED(memid);
|
||||
return true;
|
||||
#endif
|
||||
}
|
||||
|
||||
// In secure mode, try to decommit an area and output a warning if this fails.
|
||||
bool _mi_os_secure_guard_page_set_before(void* addr, mi_memid_t memid) {
|
||||
return _mi_os_secure_guard_page_set_at((uint8_t*)addr - _mi_os_secure_guard_page_size(), memid);
|
||||
}
|
||||
|
||||
// In secure mode, try to recommit an area
|
||||
bool _mi_os_secure_guard_page_reset_at(void* addr, mi_memid_t memid) {
|
||||
if (addr == NULL) return true;
|
||||
#if MI_SECURE > 0
|
||||
if (!memid.is_pinned) {
|
||||
mi_arena_t* const arena = mi_memid_arena(memid);
|
||||
if (arena != NULL && arena->commit_fun != NULL) {
|
||||
return (*(arena->commit_fun))(true, addr, _mi_os_secure_guard_page_size(), NULL, arena->commit_fun_arg);
|
||||
}
|
||||
else {
|
||||
return _mi_os_commit(addr, _mi_os_secure_guard_page_size(), NULL);
|
||||
}
|
||||
}
|
||||
#else
|
||||
MI_UNUSED(memid);
|
||||
#endif
|
||||
return true;
|
||||
}
|
||||
|
||||
// In secure mode, try to recommit an area
|
||||
bool _mi_os_secure_guard_page_reset_before(void* addr, mi_memid_t memid) {
|
||||
return _mi_os_secure_guard_page_reset_at((uint8_t*)addr - _mi_os_secure_guard_page_size(), memid);
|
||||
}
|
||||
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Free memory
|
||||
-------------------------------------------------------------- */
|
||||
|
||||
static void mi_os_free_huge_os_pages(void* p, size_t size, mi_subproc_t* subproc);
|
||||
|
||||
static void mi_os_prim_free(void* addr, size_t size, size_t commit_size, mi_subproc_t* subproc) {
|
||||
mi_assert_internal((size % _mi_os_page_size()) == 0);
|
||||
if (addr == NULL) return; // || _mi_os_is_huge_reserved(addr)
|
||||
int err = _mi_prim_free(addr, size); // allow size==0 (issue #1041)
|
||||
if (err != 0) {
|
||||
_mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr);
|
||||
}
|
||||
if (subproc == NULL) { subproc = _mi_subproc(); } // from `mi_arenas_unsafe_destroy` we pass subproc_main explicitly as we can no longer use the theap pointer
|
||||
if (commit_size > 0) {
|
||||
mi_subproc_stat_decrease(subproc, committed, commit_size);
|
||||
}
|
||||
mi_subproc_stat_decrease(subproc, reserved, size);
|
||||
}
|
||||
|
||||
void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid, mi_subproc_t* subproc /* can be NULL */) {
|
||||
if (mi_memkind_is_os(memid.memkind)) {
|
||||
size_t csize = memid.mem.os.size;
|
||||
if (csize==0) { csize = _mi_os_good_alloc_size(size); }
|
||||
mi_assert_internal(csize >= size);
|
||||
size_t commit_size = (still_committed ? csize : 0);
|
||||
void* base = addr;
|
||||
// different base? (due to alignment)
|
||||
if (memid.mem.os.base != base) {
|
||||
mi_assert(memid.mem.os.base <= addr);
|
||||
base = memid.mem.os.base;
|
||||
const size_t diff = (uint8_t*)addr - (uint8_t*)memid.mem.os.base;
|
||||
if (memid.mem.os.size==0) {
|
||||
csize += diff;
|
||||
}
|
||||
if (still_committed) {
|
||||
commit_size -= diff; // the (addr-base) part was already un-committed
|
||||
}
|
||||
}
|
||||
// free it
|
||||
if (memid.memkind == MI_MEM_OS_HUGE) {
|
||||
mi_assert(memid.is_pinned);
|
||||
mi_os_free_huge_os_pages(base, csize, subproc);
|
||||
}
|
||||
else {
|
||||
mi_os_prim_free(base, csize, (still_committed ? commit_size : 0), subproc);
|
||||
}
|
||||
}
|
||||
else {
|
||||
// nothing to do
|
||||
mi_assert(memid.memkind < MI_MEM_OS);
|
||||
}
|
||||
}
|
||||
|
||||
void _mi_os_free(void* p, size_t size, mi_memid_t memid) {
|
||||
_mi_os_free_ex(p, size, true, memid, NULL);
|
||||
}
|
||||
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Primitive allocation from the OS.
|
||||
-------------------------------------------------------------- */
|
||||
|
||||
// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
|
||||
// Also `hint_addr` is a hint and may be ignored.
|
||||
static void* mi_os_prim_alloc_at(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {
|
||||
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
|
||||
mi_assert_internal(is_zero != NULL);
|
||||
mi_assert_internal(is_large != NULL);
|
||||
if (size == 0) return NULL;
|
||||
if (!commit) { allow_large = false; }
|
||||
if (try_alignment == 0) { try_alignment = 1; } // avoid 0 to ensure there will be no divide by zero when aligning
|
||||
|
||||
// try to align along large OS page size for larger allocations
|
||||
const size_t large_page_size = mi_os_mem_config.large_page_size;
|
||||
if (large_page_size > 0 && hint_addr == NULL && size >= 8*large_page_size && _mi_is_power_of_two(try_alignment) && try_alignment < large_page_size) {
|
||||
try_alignment = large_page_size;
|
||||
}
|
||||
|
||||
*is_zero = false;
|
||||
void* p = NULL;
|
||||
int err = _mi_prim_alloc(hint_addr, size, try_alignment, commit, allow_large, is_large, is_zero, &p);
|
||||
if (err != 0) {
|
||||
_mi_warning_message("unable to allocate OS memory (error: %d (0x%x), addr: %p, size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, hint_addr, size, try_alignment, commit, allow_large);
|
||||
}
|
||||
|
||||
mi_os_stat_counter_increase(mmap_calls, 1);
|
||||
if (p != NULL) {
|
||||
mi_os_stat_increase(reserved, size);
|
||||
if (commit) {
|
||||
mi_os_stat_increase(committed, size);
|
||||
// seems needed for asan (or `mimalloc-test-api` fails)
|
||||
#ifdef MI_TRACK_ASAN
|
||||
if (*is_zero) { mi_track_mem_defined(p,size); }
|
||||
else { mi_track_mem_undefined(p,size); }
|
||||
#endif
|
||||
}
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {
|
||||
return mi_os_prim_alloc_at(NULL, size, try_alignment, commit, allow_large, is_large, is_zero);
|
||||
}
|
||||
|
||||
|
||||
// Primitive aligned allocation from the OS.
|
||||
// This function guarantees the allocated memory is aligned.
|
||||
static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid) {
|
||||
mi_assert_internal(memid!=NULL);
|
||||
mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0));
|
||||
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
|
||||
*memid = _mi_memid_none();
|
||||
if (!commit) allow_large = false;
|
||||
if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL;
|
||||
size = _mi_align_up(size, _mi_os_page_size());
|
||||
|
||||
// try a direct allocation if the alignment is below the default, or less than or equal to 1/4 fraction of the size.
|
||||
const bool try_direct_alloc = (alignment <= mi_os_mem_config.alloc_granularity || alignment <= size/4);
|
||||
|
||||
bool os_is_large = false;
|
||||
bool os_is_zero = false;
|
||||
void* os_base = NULL;
|
||||
size_t os_size = size;
|
||||
void* p = NULL;
|
||||
if (try_direct_alloc) {
|
||||
p = mi_os_prim_alloc(size, alignment, commit, allow_large, &os_is_large, &os_is_zero);
|
||||
}
|
||||
|
||||
// aligned already?
|
||||
if (p != NULL && _mi_is_aligned(p,alignment)) {
|
||||
os_base = p;
|
||||
}
|
||||
else {
|
||||
// if not aligned, free it, overallocate, and unmap around it
|
||||
#if !MI_TRACK_ASAN
|
||||
if (try_direct_alloc) {
|
||||
_mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit);
|
||||
}
|
||||
#endif
|
||||
if (p != NULL) { mi_os_prim_free(p, size, (commit ? size : 0), NULL); }
|
||||
if (size >= (SIZE_MAX - alignment)) return NULL; // overflow
|
||||
const size_t over_size = size + alignment;
|
||||
|
||||
if (!mi_os_mem_config.has_partial_free) { // win32 virtualAlloc cannot free parts of an allocated block
|
||||
// over-allocate uncommitted (virtual) memory
|
||||
p = mi_os_prim_alloc(over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, &os_is_large, &os_is_zero);
|
||||
if (p == NULL) return NULL;
|
||||
|
||||
// set p to the aligned part in the full region
|
||||
// note: Windows VirtualFree needs the actual base pointer
|
||||
// this is handled though by having the `base` field in the memid
|
||||
os_base = p; // remember the base
|
||||
os_size = over_size;
|
||||
p = _mi_align_up_ptr(p, alignment);
|
||||
|
||||
// explicitly commit only the aligned part
|
||||
if (commit) {
|
||||
if (!_mi_os_commit(p, size, NULL)) {
|
||||
mi_os_prim_free(os_base, over_size, 0, NULL);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
else { // mmap can free inside an allocation
|
||||
// overallocate...
|
||||
p = mi_os_prim_alloc(over_size, 1, commit, false, &os_is_large, &os_is_zero);
|
||||
if (p == NULL) return NULL;
|
||||
|
||||
// and selectively unmap parts around the over-allocated area.
|
||||
void* const aligned_p = _mi_align_up_ptr(p, alignment);
|
||||
const size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p;
|
||||
const size_t mid_size = _mi_align_up(size, _mi_os_page_size());
|
||||
const size_t post_size = over_size - pre_size - mid_size;
|
||||
mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size);
|
||||
if (pre_size > 0) { mi_os_prim_free(p, pre_size, (commit ? pre_size : 0), NULL); }
|
||||
if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0), NULL); }
|
||||
// we can return the aligned pointer on `mmap` systems
|
||||
p = aligned_p;
|
||||
os_base = aligned_p; // since we freed the pre part, `*base == p`.
|
||||
os_size = mid_size;
|
||||
}
|
||||
}
|
||||
|
||||
mi_assert_internal(p != NULL && os_base != NULL && _mi_is_aligned(p,alignment));
|
||||
mi_assert_internal(os_base <= p && size <= os_size);
|
||||
*memid = _mi_memid_create_os(os_base,os_size,commit,os_is_zero,os_is_large);
|
||||
return p;
|
||||
}
|
||||
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
OS API: alloc and alloc_aligned
|
||||
----------------------------------------------------------- */
|
||||
|
||||
void* _mi_os_alloc(size_t size, mi_memid_t* memid) {
|
||||
*memid = _mi_memid_none();
|
||||
if (size == 0) return NULL;
|
||||
size = _mi_os_good_alloc_size(size);
|
||||
bool os_is_large = false;
|
||||
bool os_is_zero = false;
|
||||
void* p = mi_os_prim_alloc(size, 0, true, false, &os_is_large, &os_is_zero);
|
||||
if (p == NULL) return NULL;
|
||||
|
||||
*memid = _mi_memid_create_os(p, size, true, os_is_zero, os_is_large);
|
||||
mi_assert_internal(memid->mem.os.size >= size);
|
||||
mi_assert_internal(memid->initially_committed);
|
||||
return p;
|
||||
}
|
||||
|
||||
void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid)
|
||||
{
|
||||
MI_UNUSED(&_mi_os_get_aligned_hint); // suppress unused warnings
|
||||
*memid = _mi_memid_none();
|
||||
if (size == 0) return NULL;
|
||||
size = _mi_os_good_alloc_size(size);
|
||||
alignment = _mi_align_up(alignment, _mi_os_page_size());
|
||||
|
||||
void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, memid );
|
||||
if (p == NULL) return NULL;
|
||||
|
||||
mi_assert_internal(memid->mem.os.size >= size);
|
||||
mi_assert_internal(_mi_is_aligned(p,alignment));
|
||||
if (commit) { mi_assert_internal(memid->initially_committed); }
|
||||
return p;
|
||||
}
|
||||
|
||||
|
||||
mi_decl_nodiscard static void* mi_os_ensure_zero(void* p, size_t size, mi_memid_t* memid) {
|
||||
if (p==NULL || size==0) return p;
|
||||
// ensure committed
|
||||
if (!memid->initially_committed) {
|
||||
bool is_zero = false;
|
||||
if (!_mi_os_commit(p, size, &is_zero)) {
|
||||
_mi_os_free(p, size, *memid);
|
||||
return NULL;
|
||||
}
|
||||
memid->initially_committed = true;
|
||||
}
|
||||
// ensure zero'd
|
||||
if (memid->initially_zero) return p;
|
||||
_mi_memzero_aligned(p,size);
|
||||
memid->initially_zero = true;
|
||||
return p;
|
||||
}
|
||||
|
||||
void* _mi_os_zalloc(size_t size, mi_memid_t* memid) {
|
||||
void* p = _mi_os_alloc(size,memid);
|
||||
return mi_os_ensure_zero(p, size, memid);
|
||||
}
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
OS aligned allocation with an offset. This is used
|
||||
for large alignments > MI_BLOCK_ALIGNMENT_MAX. We use a large mimalloc
|
||||
page where the object can be aligned at an offset from the start of the segment.
|
||||
As we may need to overallocate, we need to free such pointers using `mi_free_aligned`
|
||||
to use the actual start of the memory region.
|
||||
----------------------------------------------------------- */
|
||||
|
||||
void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offset, bool commit, bool allow_large, mi_memid_t* memid) {
|
||||
mi_assert(offset <= size);
|
||||
mi_assert((alignment % _mi_os_page_size()) == 0);
|
||||
*memid = _mi_memid_none();
|
||||
if (offset > size) return NULL;
|
||||
if (offset == 0) {
|
||||
// regular aligned allocation
|
||||
return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid);
|
||||
}
|
||||
else {
|
||||
// overallocate to align at an offset
|
||||
const size_t extra = _mi_align_up(offset, alignment) - offset;
|
||||
if (size >= SIZE_MAX - extra) return NULL; // too large
|
||||
const size_t oversize = size + extra;
|
||||
void* const start = _mi_os_alloc_aligned(oversize, alignment, commit, allow_large, memid);
|
||||
if (start == NULL) return NULL;
|
||||
|
||||
void* const p = (uint8_t*)start + extra;
|
||||
mi_assert(_mi_is_aligned((uint8_t*)p + offset, alignment));
|
||||
// decommit the overallocation at the start
|
||||
if (commit && extra >= _mi_os_page_size()) {
|
||||
_mi_os_decommit(start, extra);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
}
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
OS memory API: reset, commit, decommit, protect, unprotect.
|
||||
----------------------------------------------------------- */
|
||||
|
||||
// OS page align within a given area, either conservative (pages inside the area only),
|
||||
// or not (straddling pages outside the area is possible)
|
||||
static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size, size_t* newsize) {
|
||||
mi_assert(addr != NULL && size > 0);
|
||||
if (newsize != NULL) *newsize = 0;
|
||||
if (size == 0 || addr == NULL) return NULL;
|
||||
|
||||
// page align conservatively within the range, or liberally straddling pages outside the range
|
||||
void* start = (conservative ? _mi_align_up_ptr(addr, _mi_os_page_size())
|
||||
: _mi_align_down_ptr(addr, _mi_os_page_size()));
|
||||
void* end = (conservative ? _mi_align_down_ptr((uint8_t*)addr + size, _mi_os_page_size())
|
||||
: _mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size()));
|
||||
ptrdiff_t diff = (uint8_t*)end - (uint8_t*)start;
|
||||
if (diff <= 0) return NULL;
|
||||
|
||||
mi_assert_internal((conservative && (size_t)diff <= size) || (!conservative && (size_t)diff >= size));
|
||||
if (newsize != NULL) *newsize = (size_t)diff;
|
||||
return start;
|
||||
}
|
||||
|
||||
static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* newsize) {
|
||||
return mi_os_page_align_areax(true, addr, size, newsize);
|
||||
}
|
||||
|
||||
bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size) {
|
||||
if (is_zero != NULL) { *is_zero = false; }
|
||||
mi_os_stat_counter_increase(commit_calls, 1);
|
||||
|
||||
// page align range
|
||||
size_t csize;
|
||||
void* start = mi_os_page_align_areax(false /* conservative? */, addr, size, &csize);
|
||||
if (csize == 0) return true;
|
||||
|
||||
// commit
|
||||
bool os_is_zero = false;
|
||||
int err = _mi_prim_commit(start, csize, &os_is_zero);
|
||||
if (err != 0) {
|
||||
_mi_warning_message("cannot commit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
|
||||
return false;
|
||||
}
|
||||
if (os_is_zero && is_zero != NULL) {
|
||||
*is_zero = true;
|
||||
mi_assert_expensive(mi_mem_is_zero(start, csize));
|
||||
}
|
||||
// note: the following seems required for asan (otherwise `mimalloc-test-stress` fails)
|
||||
#ifdef MI_TRACK_ASAN
|
||||
if (os_is_zero) { mi_track_mem_defined(start,csize); }
|
||||
else { mi_track_mem_undefined(start,csize); }
|
||||
#endif
|
||||
mi_os_stat_increase(committed, stat_size); // use size for precise commit vs. decommit
|
||||
return true;
|
||||
}
|
||||
|
||||
bool _mi_os_commit(void* addr, size_t size, bool* is_zero) {
|
||||
return _mi_os_commit_ex(addr, size, is_zero, size);
|
||||
}
|
||||
|
||||
static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, size_t stat_size) {
|
||||
mi_assert_internal(needs_recommit!=NULL);
|
||||
|
||||
// page align
|
||||
size_t csize;
|
||||
void* start = mi_os_page_align_area_conservative(addr, size, &csize);
|
||||
if (csize == 0) return true;
|
||||
|
||||
// decommit
|
||||
*needs_recommit = true;
|
||||
int err = _mi_prim_decommit(start,csize,needs_recommit);
|
||||
if (err != 0) {
|
||||
_mi_warning_message("cannot decommit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
|
||||
}
|
||||
else if (*needs_recommit) {
|
||||
mi_os_stat_decrease(committed, stat_size);
|
||||
}
|
||||
mi_assert_internal(err == 0);
|
||||
return (err == 0);
|
||||
}
|
||||
|
||||
bool _mi_os_decommit(void* addr, size_t size) {
|
||||
bool needs_recommit;
|
||||
return mi_os_decommit_ex(addr, size, &needs_recommit, size);
|
||||
}
|
||||
|
||||
|
||||
// Signal to the OS that the address range is no longer in use
|
||||
// but may be used later again. This will release physical memory
|
||||
// pages and reduce swapping while keeping the memory committed.
|
||||
// We page align to a conservative area inside the range to reset.
|
||||
bool _mi_os_reset(void* addr, size_t size) {
|
||||
// page align conservatively within the range
|
||||
size_t csize;
|
||||
void* start = mi_os_page_align_area_conservative(addr, size, &csize);
|
||||
if (csize == 0) return true; // || _mi_os_is_huge_reserved(addr)
|
||||
mi_os_stat_counter_increase(reset, csize);
|
||||
mi_os_stat_counter_increase(reset_calls, 1);
|
||||
|
||||
#if (MI_DEBUG>1) && !MI_SECURE && !MI_TRACK_ENABLED // && !MI_TSAN
|
||||
memset(start, 0, csize); // pretend it is eagerly reset
|
||||
#endif
|
||||
|
||||
int err = _mi_prim_reset(start, csize);
|
||||
if (err != 0) {
|
||||
_mi_warning_message("cannot reset OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
|
||||
}
|
||||
return (err == 0);
|
||||
}
|
||||
|
||||
|
||||
void _mi_os_reuse( void* addr, size_t size ) {
|
||||
// page align conservatively within the range
|
||||
size_t csize = 0;
|
||||
void* const start = mi_os_page_align_area_conservative(addr, size, &csize);
|
||||
if (csize == 0) return;
|
||||
const int err = _mi_prim_reuse(start, csize);
|
||||
if (err != 0) {
|
||||
_mi_warning_message("cannot reuse OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
|
||||
}
|
||||
}
|
||||
|
||||
// either resets or decommits memory, returns true if the memory needs
|
||||
// to be recommitted if it is to be re-used later on.
|
||||
bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg)
|
||||
{
|
||||
if (mi_option_get(mi_option_purge_delay) < 0) return false; // is purging allowed?
|
||||
mi_os_stat_counter_increase(purge_calls, 1);
|
||||
mi_os_stat_counter_increase(purged, size);
|
||||
|
||||
if (commit_fun != NULL) {
|
||||
bool decommitted = (*commit_fun)(false, p, size, NULL, commit_fun_arg);
|
||||
return decommitted; // needs_recommit?
|
||||
}
|
||||
else if (mi_option_is_enabled(mi_option_purge_decommits) && // should decommit?
|
||||
!_mi_preloading()) // don't decommit during preloading (unsafe)
|
||||
{
|
||||
bool needs_recommit = true;
|
||||
mi_os_decommit_ex(p, size, &needs_recommit, stat_size);
|
||||
return needs_recommit;
|
||||
}
|
||||
else {
|
||||
if (allow_reset) { // this can sometimes be not allowed if the range is not fully committed (on Windows, we cannot reset uncommitted memory)
|
||||
_mi_os_reset(p, size);
|
||||
}
|
||||
return false; // needs no recommit
|
||||
}
|
||||
}
|
||||
|
||||
// either resets or decommits memory, returns true if the memory needs
|
||||
// to be recommitted if it is to be re-used later on.
|
||||
bool _mi_os_purge(void* p, size_t size) {
|
||||
return _mi_os_purge_ex(p, size, true, size, NULL, NULL);
|
||||
}
|
||||
|
||||
|
||||
// Protect a region in memory to be not accessible.
|
||||
static bool mi_os_protectx(void* addr, size_t size, bool protect) {
|
||||
// page align conservatively within the range
|
||||
size_t csize = 0;
|
||||
void* start = mi_os_page_align_area_conservative(addr, size, &csize);
|
||||
if (csize == 0) return false;
|
||||
/*
|
||||
if (_mi_os_is_huge_reserved(addr)) {
|
||||
_mi_warning_message("cannot mprotect memory allocated in huge OS pages\n");
|
||||
}
|
||||
*/
|
||||
int err = _mi_prim_protect(start,csize,protect);
|
||||
if (err != 0) {
|
||||
_mi_warning_message("cannot %s OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", (protect ? "protect" : "unprotect"), err, err, start, csize);
|
||||
}
|
||||
return (err == 0);
|
||||
}
|
||||
|
||||
bool _mi_os_protect(void* addr, size_t size) {
|
||||
return mi_os_protectx(addr, size, true);
|
||||
}
|
||||
|
||||
bool _mi_os_unprotect(void* addr, size_t size) {
|
||||
return mi_os_protectx(addr, size, false);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
Support for allocating huge OS pages (1Gib) that are reserved up-front
|
||||
and possibly associated with a specific NUMA node. (use `numa_node>=0`)
|
||||
-----------------------------------------------------------------------------*/
|
||||
#define MI_HUGE_OS_PAGE_SIZE (MI_GiB)
|
||||
|
||||
|
||||
#if (MI_INTPTR_SIZE >= 8)
|
||||
// To ensure proper alignment, use our own area for huge OS pages
|
||||
static mi_decl_cache_align _Atomic(uintptr_t) mi_huge_start; // = 0
|
||||
|
||||
// Claim an aligned address range for huge pages
|
||||
static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
|
||||
if (total_size != NULL) *total_size = 0;
|
||||
const size_t size = pages * MI_HUGE_OS_PAGE_SIZE;
|
||||
|
||||
uintptr_t start = 0;
|
||||
uintptr_t end = 0;
|
||||
uintptr_t huge_start = mi_atomic_load_relaxed(&mi_huge_start);
|
||||
do {
|
||||
start = huge_start;
|
||||
if (start == 0) {
|
||||
// Initialize the start address after the 32TiB area
|
||||
start = ((uintptr_t)8 << 40); // 8TiB virtual start address
|
||||
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode
|
||||
mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267)
|
||||
if (mi_theap_is_initialized(theap)) { // todo: or no hint at all if we lack randomness?
|
||||
const uintptr_t r = _mi_theap_random_next(theap);
|
||||
start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB
|
||||
}
|
||||
else {
|
||||
_mi_warning_message("failed to randomize the start address of huge pages allocation (%zu bytes at %p)", size, start);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
end = start + size;
|
||||
} while (!mi_atomic_cas_weak_acq_rel(&mi_huge_start, &huge_start, end));
|
||||
|
||||
if (total_size != NULL) *total_size = size;
|
||||
return (uint8_t*)start;
|
||||
}
|
||||
#else
|
||||
static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
|
||||
MI_UNUSED(pages);
|
||||
if (total_size != NULL) *total_size = 0;
|
||||
return NULL;
|
||||
}
|
||||
#endif
|
||||
|
||||
// Allocate MI_ARENA_SLICE_ALIGN aligned huge pages
|
||||
void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_msecs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid) {
|
||||
*memid = _mi_memid_none();
|
||||
if (psize != NULL) *psize = 0;
|
||||
if (pages_reserved != NULL) *pages_reserved = 0;
|
||||
size_t size = 0;
|
||||
uint8_t* const start = mi_os_claim_huge_pages(pages, &size);
|
||||
if (start == NULL) return NULL; // or 32-bit systems
|
||||
|
||||
// Allocate one page at the time but try to place them contiguously
|
||||
// We allocate one page at the time to be able to abort if it takes too long
|
||||
// or to at least allocate as many as available on the system.
|
||||
mi_msecs_t start_t = _mi_clock_start();
|
||||
size_t page = 0;
|
||||
bool all_zero = true;
|
||||
while (page < pages) {
|
||||
// allocate a page
|
||||
bool is_zero = false;
|
||||
void* addr = start + (page * MI_HUGE_OS_PAGE_SIZE);
|
||||
void* p = NULL;
|
||||
int err = _mi_prim_alloc_huge_os_pages(addr, MI_HUGE_OS_PAGE_SIZE, numa_node, &is_zero, &p);
|
||||
if (!is_zero) { all_zero = false; }
|
||||
if (err != 0) {
|
||||
_mi_warning_message("unable to allocate huge OS page (error: %d (0x%x), address: %p, size: %zx bytes)\n", err, err, addr, MI_HUGE_OS_PAGE_SIZE);
|
||||
break;
|
||||
}
|
||||
|
||||
// Did we succeed at a contiguous address?
|
||||
if (p != addr) {
|
||||
// no success, issue a warning and break
|
||||
if (p != NULL) {
|
||||
_mi_warning_message("could not allocate contiguous huge OS page %zu at %p\n", page, addr);
|
||||
mi_os_prim_free(p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, NULL);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
// success, record it
|
||||
page++; // increase before timeout check (see issue #711)
|
||||
mi_os_stat_increase(committed, MI_HUGE_OS_PAGE_SIZE);
|
||||
mi_os_stat_increase(reserved, MI_HUGE_OS_PAGE_SIZE);
|
||||
|
||||
// check for timeout
|
||||
if (max_msecs > 0) {
|
||||
mi_msecs_t elapsed = _mi_clock_end(start_t);
|
||||
if (page >= 1) {
|
||||
mi_msecs_t estimate = ((elapsed / (page+1)) * pages);
|
||||
if (estimate > 2*max_msecs) { // seems like we are going to timeout, break
|
||||
elapsed = max_msecs + 1;
|
||||
}
|
||||
}
|
||||
if (elapsed > max_msecs) {
|
||||
_mi_warning_message("huge OS page allocation timed out (after allocating %zu page(s))\n", page);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
const size_t allocated = page * MI_HUGE_OS_PAGE_SIZE;
|
||||
mi_assert_internal(allocated <= size);
|
||||
if (pages_reserved != NULL) { *pages_reserved = page; }
|
||||
if (psize != NULL) { *psize = allocated; }
|
||||
if (page != 0) {
|
||||
mi_assert(start != NULL);
|
||||
*memid = _mi_memid_create_os(start, allocated, true /* is committed */, all_zero, true /* is_large */);
|
||||
memid->memkind = MI_MEM_OS_HUGE;
|
||||
mi_assert(memid->is_pinned);
|
||||
#ifdef MI_TRACK_ASAN
|
||||
if (all_zero) { mi_track_mem_defined(start,allocated); }
|
||||
#endif
|
||||
}
|
||||
return (page == 0 ? NULL : start);
|
||||
}
|
||||
|
||||
// free every huge page in a range individually (as we allocated per page)
|
||||
// note: needed with VirtualAlloc but could potentially be done in one go on mmap'd systems.
|
||||
static void mi_os_free_huge_os_pages(void* p, size_t size, mi_subproc_t* subproc) {
|
||||
if (p==NULL || size==0) return;
|
||||
uint8_t* base = (uint8_t*)p;
|
||||
while (size >= MI_HUGE_OS_PAGE_SIZE) {
|
||||
mi_os_prim_free(base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, subproc);
|
||||
size -= MI_HUGE_OS_PAGE_SIZE;
|
||||
base += MI_HUGE_OS_PAGE_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
Support NUMA aware allocation
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
static _Atomic(size_t) mi_numa_node_count; // = 0 // cache the node count
|
||||
|
||||
int _mi_os_numa_node_count(void) {
|
||||
size_t count = mi_atomic_load_acquire(&mi_numa_node_count);
|
||||
if mi_unlikely(count == 0) {
|
||||
long ncount = mi_option_get(mi_option_use_numa_nodes); // given explicitly?
|
||||
if (ncount > 0 && ncount < INT_MAX) {
|
||||
count = (size_t)ncount;
|
||||
}
|
||||
else {
|
||||
const size_t n = _mi_prim_numa_node_count(); // or detect dynamically
|
||||
if (n == 0 || n > INT_MAX) { count = 1; }
|
||||
else { count = n; }
|
||||
}
|
||||
mi_atomic_store_release(&mi_numa_node_count, count); // save it
|
||||
if (count>1) { _mi_verbose_message("using %zd numa regions\n", count); }
|
||||
}
|
||||
mi_assert_internal(count > 0 && count <= INT_MAX);
|
||||
return (int)count;
|
||||
}
|
||||
|
||||
static int mi_os_numa_node_get(void) {
|
||||
int numa_count = _mi_os_numa_node_count();
|
||||
if (numa_count<=1) return 0; // optimize on single numa node systems: always node 0
|
||||
// never more than the node count and >= 0
|
||||
const size_t n = _mi_prim_numa_node();
|
||||
int numa_node = (n < INT_MAX ? (int)n : 0);
|
||||
if (numa_node >= numa_count) { numa_node = numa_node % numa_count; }
|
||||
return numa_node;
|
||||
}
|
||||
|
||||
int _mi_os_numa_node(void) {
|
||||
if mi_likely(mi_atomic_load_relaxed(&mi_numa_node_count) == 1) {
|
||||
return 0;
|
||||
}
|
||||
else {
|
||||
return mi_os_numa_node_get();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
Public API
|
||||
-----------------------------------------------------------------------------*/
|
||||
#if 0
|
||||
mi_decl_export void* mi_os_alloc(size_t size, bool commit, size_t* full_size) {
|
||||
return mi_os_alloc_aligned(size, mi_os_mem_config.alloc_granularity, commit, NULL, full_size);
|
||||
}
|
||||
|
||||
static void* mi_os_alloc_aligned_ex(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_committed, bool* is_pinned, void** base, size_t* full_size) {
|
||||
mi_memid_t memid = _mi_memid_none();
|
||||
void* p = _mi_os_alloc_aligned(size, alignment, commit, allow_large, &memid);
|
||||
if (p == NULL) return p;
|
||||
if (is_committed != NULL) { *is_committed = memid.initially_committed; }
|
||||
if (is_pinned != NULL) { *is_pinned = memid.is_pinned; }
|
||||
if (base != NULL) { *base = memid.mem.os.base; }
|
||||
if (full_size != NULL) { *full_size = memid.mem.os.size; }
|
||||
if (!memid.initially_zero && memid.initially_committed) {
|
||||
_mi_memzero_aligned(memid.mem.os.base, memid.mem.os.size);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
mi_decl_export void* mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, void** base, size_t* full_size) {
|
||||
return mi_os_alloc_aligned_ex(size, alignment, commit, false, NULL, NULL, base, full_size);
|
||||
}
|
||||
|
||||
mi_decl_export void* mi_os_alloc_aligned_allow_large(size_t size, size_t alignment, bool commit, bool* is_committed, bool* is_pinned, void** base, size_t* full_size) {
|
||||
return mi_os_alloc_aligned_ex(size, alignment, commit, true, is_committed, is_pinned, base, full_size);
|
||||
}
|
||||
|
||||
mi_decl_export void mi_os_free(void* p, size_t size) {
|
||||
if (p==NULL || size == 0) return;
|
||||
mi_memid_t memid = _mi_memid_create_os(p, size, true, false, false);
|
||||
_mi_os_free(p, size, memid);
|
||||
}
|
||||
|
||||
mi_decl_export void mi_os_commit(void* p, size_t size) {
|
||||
_mi_os_commit(p, size, NULL);
|
||||
}
|
||||
|
||||
mi_decl_export void mi_os_decommit(void* p, size_t size) {
|
||||
_mi_os_decommit(p, size);
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,447 @@
|
||||
/*----------------------------------------------------------------------------
|
||||
Copyright (c) 2023-2025, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc/internal.h"
|
||||
#include "bitmap.h"
|
||||
|
||||
static void mi_page_map_cannot_commit(void) {
|
||||
_mi_warning_message("unable to commit the allocation page-map on-demand\n" );
|
||||
}
|
||||
|
||||
#if MI_PAGE_MAP_FLAT
|
||||
|
||||
// The page-map contains a byte for each 64kb slice in the address space.
|
||||
// For an address `a` where `ofs = _mi_page_map[a >> 16]`:
|
||||
// 0 = unused
|
||||
// 1 = the slice at `a & ~0xFFFF` is a mimalloc page.
|
||||
// 1 < ofs <= 127 = the slice is part of a page, starting at `(((a>>16) - ofs - 1) << 16)`.
|
||||
//
|
||||
// 1 byte per slice => 1 TiB address space needs a 2^14 * 2^16 = 16 MiB page map.
|
||||
// A full 256 TiB address space (48 bit) needs a 4 GiB page map.
|
||||
// A full 4 GiB address space (32 bit) needs only a 64 KiB page map.
|
||||
|
||||
mi_decl_cache_align uint8_t* _mi_page_map = NULL;
|
||||
static void* mi_page_map_max_address = NULL;
|
||||
static mi_memid_t mi_page_map_memid;
|
||||
|
||||
#define MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT MI_ARENA_SLICE_SIZE
|
||||
static mi_bitmap_t* mi_page_map_commit; // one bit per committed 64 KiB entries
|
||||
|
||||
mi_decl_nodiscard static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count);
|
||||
|
||||
bool _mi_page_map_init(void) {
|
||||
size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_SIZE_BITS);
|
||||
if (vbits == 0) {
|
||||
vbits = _mi_os_virtual_address_bits();
|
||||
#if MI_ARCH_X64 // canonical address is limited to the first 128 TiB
|
||||
if (vbits >= 48) { vbits = 47; }
|
||||
#endif
|
||||
}
|
||||
if (vbits < MI_ARENA_SLICE_SHIFT) {
|
||||
vbits = MI_ARENA_SLICE_SHIFT;
|
||||
}
|
||||
|
||||
// Allocate the page map and commit bits
|
||||
mi_page_map_max_address = (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits));
|
||||
const size_t page_map_size = (MI_ZU(1) << (vbits - MI_ARENA_SLICE_SHIFT));
|
||||
const bool commit = (page_map_size <= 1*MI_MiB || mi_option_is_enabled(mi_option_pagemap_commit)); // _mi_os_has_overcommit(); // commit on-access on Linux systems?
|
||||
const size_t commit_bits = _mi_divide_up(page_map_size, MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT);
|
||||
const size_t bitmap_size = (commit ? 0 : mi_bitmap_size(commit_bits, NULL));
|
||||
const size_t reserve_size = bitmap_size + page_map_size;
|
||||
uint8_t* const base = (uint8_t*)_mi_os_alloc_aligned(reserve_size, 1, commit, true /* allow large */, &mi_page_map_memid);
|
||||
if (base==NULL) {
|
||||
_mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", page_map_size / MI_KiB);
|
||||
return false;
|
||||
}
|
||||
if (mi_page_map_memid.initially_committed && !mi_page_map_memid.initially_zero) {
|
||||
_mi_warning_message("internal: the page map was committed but not zero initialized!\n");
|
||||
_mi_memzero_aligned(base, reserve_size);
|
||||
}
|
||||
if (bitmap_size > 0) {
|
||||
mi_page_map_commit = (mi_bitmap_t*)base;
|
||||
if (!_mi_os_commit(mi_page_map_commit, bitmap_size, NULL)) {
|
||||
mi_page_map_cannot_commit();
|
||||
return false;
|
||||
}
|
||||
mi_bitmap_init(mi_page_map_commit, commit_bits, true);
|
||||
}
|
||||
_mi_page_map = base + bitmap_size;
|
||||
|
||||
// commit the first part so NULL pointers get resolved without an access violation
|
||||
if (!commit) {
|
||||
if (!mi_page_map_ensure_committed(0, 1)) {
|
||||
mi_page_map_cannot_commit();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
_mi_page_map[0] = 1; // so _mi_ptr_page(NULL) == NULL
|
||||
mi_assert_internal(_mi_ptr_page(NULL)==NULL);
|
||||
return true;
|
||||
}
|
||||
|
||||
void _mi_page_map_unsafe_destroy(mi_subproc_t* subproc) {
|
||||
mi_assert_internal(subproc != NULL);
|
||||
mi_assert_internal(_mi_page_map != NULL);
|
||||
if (_mi_page_map == NULL) return;
|
||||
_mi_os_free_ex(mi_page_map_memid.mem.os.base, mi_page_map_memid.mem.os.size, true, mi_page_map_memid, subproc);
|
||||
_mi_page_map = NULL;
|
||||
mi_page_map_commit = NULL;
|
||||
mi_page_map_max_address = NULL;
|
||||
mi_page_map_memid = _mi_memid_none();
|
||||
}
|
||||
|
||||
|
||||
static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count) {
|
||||
// is the page map area that contains the page address committed?
|
||||
// we always set the commit bits so we can track what ranges are in-use.
|
||||
// we only actually commit if the map wasn't committed fully already.
|
||||
if (mi_page_map_commit != NULL) {
|
||||
const size_t commit_idx = idx / MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT;
|
||||
const size_t commit_idx_hi = (idx + slice_count - 1) / MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT;
|
||||
for (size_t i = commit_idx; i <= commit_idx_hi; i++) { // per bit to avoid crossing over bitmap chunks
|
||||
if (mi_bitmap_is_clear(mi_page_map_commit, i)) {
|
||||
// this may race, in which case we do multiple commits (which is ok)
|
||||
bool is_zero;
|
||||
uint8_t* const start = _mi_page_map + (i * MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT);
|
||||
const size_t size = MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT;
|
||||
if (!_mi_os_commit(start, size, &is_zero)) {
|
||||
mi_page_map_cannot_commit();
|
||||
return false;
|
||||
}
|
||||
if (!is_zero && !mi_page_map_memid.initially_zero) { _mi_memzero(start, size); }
|
||||
mi_bitmap_set(mi_page_map_commit, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
#if MI_DEBUG > 0
|
||||
_mi_page_map[idx] = 0;
|
||||
_mi_page_map[idx+slice_count-1] = 0;
|
||||
#endif
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
static size_t mi_page_map_get_idx(mi_page_t* page, uint8_t** page_start, size_t* slice_count) {
|
||||
size_t page_size;
|
||||
*page_start = mi_page_area(page, &page_size);
|
||||
if (page_size > MI_LARGE_PAGE_SIZE) { page_size = MI_LARGE_PAGE_SIZE - MI_ARENA_SLICE_SIZE; } // furthest interior pointer
|
||||
*slice_count = mi_slice_count_of_size(page_size) + ((*page_start - mi_page_slice_start(page))/MI_ARENA_SLICE_SIZE); // add for large aligned blocks
|
||||
return _mi_page_map_index(page);
|
||||
}
|
||||
|
||||
bool _mi_page_map_register(mi_page_t* page) {
|
||||
mi_assert_internal(page != NULL);
|
||||
mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
|
||||
mi_assert_internal(_mi_page_map != NULL); // should be initialized before multi-thread access!
|
||||
if mi_unlikely(_mi_page_map == NULL) {
|
||||
if (!_mi_page_map_init()) return false;
|
||||
}
|
||||
mi_assert(_mi_page_map!=NULL);
|
||||
uint8_t* page_start;
|
||||
size_t slice_count;
|
||||
const size_t idx = mi_page_map_get_idx(page, &page_start, &slice_count);
|
||||
|
||||
if (!mi_page_map_ensure_committed(idx, slice_count)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// set the offsets
|
||||
for (size_t i = 0; i < slice_count; i++) {
|
||||
mi_assert_internal(i < 128);
|
||||
_mi_page_map[idx + i] = (uint8_t)(i+1);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void _mi_page_map_unregister(mi_page_t* page) {
|
||||
mi_assert_internal(_mi_page_map != NULL);
|
||||
// get index and count
|
||||
uint8_t* page_start;
|
||||
size_t slice_count;
|
||||
const size_t idx = mi_page_map_get_idx(page, &page_start, &slice_count);
|
||||
// unset the offsets
|
||||
_mi_memzero(_mi_page_map + idx, slice_count);
|
||||
}
|
||||
|
||||
void _mi_page_map_unregister_range(void* start, size_t size) {
|
||||
const size_t slice_count = _mi_divide_up(size, MI_ARENA_SLICE_SIZE);
|
||||
const uintptr_t index = _mi_page_map_index(start);
|
||||
// todo: scan the commit bits and clear only those ranges?
|
||||
if (!mi_page_map_ensure_committed(index, slice_count)) { // we commit the range in total;
|
||||
return;
|
||||
}
|
||||
_mi_memzero(&_mi_page_map[index], slice_count);
|
||||
}
|
||||
|
||||
|
||||
mi_page_t* _mi_safe_ptr_page(const void* p) {
|
||||
if mi_unlikely(p >= mi_page_map_max_address) return NULL;
|
||||
const uintptr_t idx = _mi_page_map_index(p);
|
||||
if mi_unlikely(mi_page_map_commit != NULL && !mi_bitmap_is_set(mi_page_map_commit, idx/MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT)) return NULL;
|
||||
const uintptr_t ofs = _mi_page_map[idx];
|
||||
if mi_unlikely(ofs == 0) return NULL;
|
||||
return (mi_page_t*)((((uintptr_t)p >> MI_ARENA_SLICE_SHIFT) - ofs + 1) << MI_ARENA_SLICE_SHIFT);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {
|
||||
return (_mi_safe_ptr_page(p) != NULL);
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
// A 2-level page map
|
||||
#define MI_PAGE_MAP_SUB_SIZE (MI_PAGE_MAP_SUB_COUNT * sizeof(mi_page_t*))
|
||||
#define MI_PAGE_MAP_ENTRIES_PER_CBIT (MI_PAGE_MAP_COUNT < MI_BFIELD_BITS ? 1 : (MI_PAGE_MAP_COUNT / MI_BFIELD_BITS))
|
||||
|
||||
mi_decl_cache_align _Atomic(mi_submap_t)* _mi_page_map;
|
||||
static size_t mi_page_map_count;
|
||||
static void* mi_page_map_max_address;
|
||||
static mi_memid_t mi_page_map_memid;
|
||||
static mi_lock_t mi_page_map_lock;
|
||||
|
||||
// divide the main map in 64 (`MI_BFIELD_BITS`) parts commit those parts on demand
|
||||
static _Atomic(mi_bfield_t) mi_page_map_commit;
|
||||
|
||||
mi_decl_nodiscard static inline bool mi_page_map_is_committed(size_t idx, size_t* pbit_idx) {
|
||||
mi_bfield_t commit = mi_atomic_load_relaxed(&mi_page_map_commit);
|
||||
const size_t bit_idx = idx/MI_PAGE_MAP_ENTRIES_PER_CBIT;
|
||||
mi_assert_internal(bit_idx < MI_BFIELD_BITS);
|
||||
if (pbit_idx != NULL) { *pbit_idx = bit_idx; }
|
||||
return ((commit & (MI_ZU(1) << bit_idx)) != 0);
|
||||
}
|
||||
|
||||
mi_decl_nodiscard static bool mi_page_map_ensure_committed(size_t idx, mi_submap_t* submap) {
|
||||
mi_assert_internal(submap!=NULL && *submap==NULL);
|
||||
size_t bit_idx;
|
||||
if mi_unlikely(!mi_page_map_is_committed(idx, &bit_idx)) {
|
||||
uint8_t* start = (uint8_t*)&_mi_page_map[bit_idx * MI_PAGE_MAP_ENTRIES_PER_CBIT];
|
||||
if (!_mi_os_commit(start, MI_PAGE_MAP_ENTRIES_PER_CBIT * sizeof(mi_submap_t), NULL)) {
|
||||
mi_page_map_cannot_commit();
|
||||
return false;
|
||||
}
|
||||
mi_atomic_or_acq_rel(&mi_page_map_commit, MI_ZU(1) << bit_idx);
|
||||
}
|
||||
*submap = mi_atomic_load_ptr_acquire(mi_page_t*, &_mi_page_map[idx]); // acquire _mi_page_map_at(idx);
|
||||
return true;
|
||||
}
|
||||
|
||||
// initialize the page map
|
||||
bool _mi_page_map_init(void) {
|
||||
size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_SIZE_BITS);
|
||||
if (vbits == 0) {
|
||||
vbits = _mi_os_virtual_address_bits();
|
||||
#if MI_ARCH_X64 // canonical address is limited to the first 128 TiB
|
||||
if (vbits >= 48) { vbits = 47; }
|
||||
#endif
|
||||
}
|
||||
if (vbits < MI_PAGE_MAP_SUB_SHIFT + MI_ARENA_SLICE_SHIFT) {
|
||||
vbits = MI_PAGE_MAP_SUB_SHIFT + MI_ARENA_SLICE_SHIFT;
|
||||
}
|
||||
|
||||
// Allocate the page map and commit bits
|
||||
mi_assert(MI_MAX_VABITS >= vbits);
|
||||
mi_page_map_max_address = (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits));
|
||||
mi_page_map_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT));
|
||||
mi_assert(mi_page_map_count <= MI_PAGE_MAP_COUNT);
|
||||
const size_t os_page_size = _mi_os_page_size();
|
||||
const size_t page_map_size = _mi_align_up( mi_page_map_count * sizeof(mi_page_t**), os_page_size);
|
||||
const size_t submap_size = MI_PAGE_MAP_SUB_SIZE;
|
||||
const size_t reserve_size = page_map_size + submap_size;
|
||||
#if MI_SECURE
|
||||
const bool commit = true; // the whole page map is valid and we can reliably check any pointer
|
||||
#else
|
||||
const bool commit = page_map_size <= 64*MI_KiB ||
|
||||
mi_option_is_enabled(mi_option_pagemap_commit) || _mi_os_has_overcommit();
|
||||
#endif
|
||||
_mi_page_map = (_Atomic(mi_page_t**)*)_mi_os_alloc_aligned(reserve_size, 1, commit, true /* allow large */, &mi_page_map_memid);
|
||||
if (_mi_page_map==NULL) {
|
||||
_mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", page_map_size / MI_KiB);
|
||||
return false;
|
||||
}
|
||||
if (mi_page_map_memid.initially_committed && !mi_page_map_memid.initially_zero) {
|
||||
_mi_warning_message("internal: the page map was committed but not zero initialized!\n");
|
||||
_mi_memzero_aligned(_mi_page_map, page_map_size);
|
||||
}
|
||||
mi_atomic_store_release(&mi_page_map_commit, (mi_page_map_memid.initially_committed ? ~MI_ZU(0) : MI_ZU(0)));
|
||||
|
||||
// ensure there is a submap for the NULL address
|
||||
mi_page_t** const sub0 = (mi_page_t**)((uint8_t*)_mi_page_map + page_map_size); // we reserved a submap part at the end already
|
||||
if (!mi_page_map_memid.initially_committed) {
|
||||
if (!_mi_os_commit(sub0, submap_size, NULL)) { // commit full submap (issue #1087)
|
||||
mi_page_map_cannot_commit();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (!mi_page_map_memid.initially_zero) { // initialize low addresses with NULL
|
||||
_mi_memzero_aligned(sub0, submap_size);
|
||||
}
|
||||
mi_submap_t nullsub = NULL;
|
||||
if (!mi_page_map_ensure_committed(0,&nullsub)) {
|
||||
mi_page_map_cannot_commit();
|
||||
return false;
|
||||
}
|
||||
mi_atomic_store_ptr_release(mi_page_t*, &_mi_page_map[0], sub0);
|
||||
mi_lock_init(&mi_page_map_lock); // initialize late in case the lock init causes allocation
|
||||
|
||||
mi_assert_internal(_mi_ptr_page(NULL)==NULL);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
void _mi_page_map_unsafe_destroy(mi_subproc_t* subproc) {
|
||||
mi_assert_internal(subproc != NULL);
|
||||
mi_assert_internal(_mi_page_map != NULL);
|
||||
if (_mi_page_map == NULL) return;
|
||||
mi_lock_done(&mi_page_map_lock);
|
||||
for (size_t idx = 1; idx < mi_page_map_count; idx++) { // skip entry 0 (as we allocate that submap at the end of the page_map)
|
||||
// free all sub-maps
|
||||
if (mi_page_map_is_committed(idx, NULL)) {
|
||||
mi_submap_t sub = _mi_page_map_at(idx);
|
||||
if (sub != NULL) {
|
||||
mi_memid_t memid = _mi_memid_create_os(sub, MI_PAGE_MAP_SUB_SIZE, true, false, false);
|
||||
_mi_os_free_ex(memid.mem.os.base, memid.mem.os.size, true, memid, subproc);
|
||||
mi_atomic_store_ptr_release(mi_page_t*, &_mi_page_map[idx], NULL);
|
||||
}
|
||||
}
|
||||
}
|
||||
_mi_os_free_ex(_mi_page_map, mi_page_map_memid.mem.os.size, true, mi_page_map_memid, subproc);
|
||||
_mi_page_map = NULL;
|
||||
mi_page_map_count = 0;
|
||||
mi_page_map_memid = _mi_memid_none();
|
||||
mi_page_map_max_address = NULL;
|
||||
mi_atomic_store_release(&mi_page_map_commit, (mi_bfield_t)0);
|
||||
}
|
||||
|
||||
|
||||
mi_decl_nodiscard static bool mi_page_map_ensure_submap_at(size_t idx, mi_submap_t* submap) {
|
||||
mi_assert_internal(submap!=NULL && *submap==NULL);
|
||||
mi_submap_t sub = NULL;
|
||||
if (!mi_page_map_ensure_committed(idx, &sub)) {
|
||||
return false;
|
||||
}
|
||||
if mi_unlikely(sub == NULL) {
|
||||
// sub map not yet allocated, alloc now
|
||||
mi_lock(&mi_page_map_lock)
|
||||
{
|
||||
sub = mi_atomic_load_ptr_acquire(mi_page_t*, &_mi_page_map[idx]); // reload
|
||||
if (sub==NULL) // not yet allocated by another thread?
|
||||
{
|
||||
mi_memid_t memid;
|
||||
const size_t submap_size = MI_PAGE_MAP_SUB_SIZE;
|
||||
sub = (mi_submap_t)_mi_os_zalloc(submap_size, &memid);
|
||||
if (sub==NULL) {
|
||||
_mi_warning_message("internal error: unable to extend the page map\n");
|
||||
}
|
||||
else {
|
||||
mi_submap_t expect = NULL;
|
||||
if (!mi_atomic_cas_ptr_strong_acq_rel(mi_page_t*, &_mi_page_map[idx], &expect, sub)) {
|
||||
// another thread already allocated it.. free and continue
|
||||
_mi_os_free(sub, submap_size, memid);
|
||||
sub = expect;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (sub==NULL) return false; // unable to allocate the submap..
|
||||
}
|
||||
mi_assert_internal(sub!=NULL);
|
||||
*submap = sub;
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool mi_page_map_set_range_prim(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) {
|
||||
// is the page map area that contains the page address committed?
|
||||
while (slice_count > 0) {
|
||||
mi_submap_t sub = NULL;
|
||||
if (!mi_page_map_ensure_submap_at(idx, &sub)) {
|
||||
return false;
|
||||
};
|
||||
mi_assert_internal(sub!=NULL);
|
||||
// set the offsets for the page
|
||||
while (slice_count > 0 && sub_idx < MI_PAGE_MAP_SUB_COUNT) {
|
||||
sub[sub_idx] = page;
|
||||
slice_count--;
|
||||
sub_idx++;
|
||||
}
|
||||
idx++; // potentially wrap around to the next idx
|
||||
sub_idx = 0;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool mi_page_map_set_range(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) {
|
||||
if mi_unlikely(!mi_page_map_set_range_prim(page,idx,sub_idx,slice_count)) {
|
||||
// failed to commit, call again to reset the page pointer if needed
|
||||
if (page!=NULL) {
|
||||
mi_page_map_set_range_prim(NULL,idx,sub_idx,slice_count);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static size_t mi_page_map_get_idx(mi_page_t* page, size_t* sub_idx, size_t* slice_count) {
|
||||
size_t page_size;
|
||||
uint8_t* page_start = mi_page_area(page, &page_size);
|
||||
if (page_size > MI_LARGE_PAGE_SIZE) { page_size = MI_LARGE_PAGE_SIZE - MI_ARENA_SLICE_SIZE; } // furthest interior pointer
|
||||
*slice_count = mi_slice_count_of_size(page_size) + ((page_start - mi_page_slice_start(page))/MI_ARENA_SLICE_SIZE); // add for large aligned blocks
|
||||
return _mi_page_map_index(page_start, sub_idx);
|
||||
}
|
||||
|
||||
bool _mi_page_map_register(mi_page_t* page) {
|
||||
mi_assert_internal(page != NULL);
|
||||
mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
|
||||
mi_assert_internal(_mi_page_map != NULL); // should be initialized before multi-thread access!
|
||||
if mi_unlikely(_mi_page_map == NULL) {
|
||||
if (!_mi_page_map_init()) return false;
|
||||
}
|
||||
mi_assert(_mi_page_map!=NULL);
|
||||
size_t slice_count;
|
||||
size_t sub_idx;
|
||||
const size_t idx = mi_page_map_get_idx(page, &sub_idx, &slice_count);
|
||||
return mi_page_map_set_range(page, idx, sub_idx, slice_count);
|
||||
}
|
||||
|
||||
void _mi_page_map_unregister(mi_page_t* page) {
|
||||
mi_assert_internal(_mi_page_map != NULL);
|
||||
mi_assert_internal(page != NULL);
|
||||
mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
|
||||
if mi_unlikely(_mi_page_map == NULL) return;
|
||||
// get index and count
|
||||
size_t slice_count;
|
||||
size_t sub_idx;
|
||||
const size_t idx = mi_page_map_get_idx(page, &sub_idx, &slice_count);
|
||||
// unset the offsets
|
||||
mi_page_map_set_range(NULL, idx, sub_idx, slice_count);
|
||||
}
|
||||
|
||||
void _mi_page_map_unregister_range(void* start, size_t size) {
|
||||
if mi_unlikely(_mi_page_map == NULL) return;
|
||||
const size_t slice_count = _mi_divide_up(size, MI_ARENA_SLICE_SIZE);
|
||||
size_t sub_idx;
|
||||
const uintptr_t idx = _mi_page_map_index(start, &sub_idx);
|
||||
mi_page_map_set_range(NULL, idx, sub_idx, slice_count); // todo: avoid committing if not already committed?
|
||||
}
|
||||
|
||||
// Return NULL for invalid pointers
|
||||
mi_page_t* _mi_safe_ptr_page(const void* p) {
|
||||
if (p==NULL) return NULL;
|
||||
if mi_unlikely(p >= mi_page_map_max_address) return NULL;
|
||||
size_t sub_idx;
|
||||
const size_t idx = _mi_page_map_index(p,&sub_idx);
|
||||
if mi_unlikely(!mi_page_map_is_committed(idx,NULL)) return NULL;
|
||||
mi_page_t** const sub = _mi_page_map[idx];
|
||||
if mi_unlikely(sub==NULL) return NULL;
|
||||
return sub[sub_idx];
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {
|
||||
return (_mi_safe_ptr_page(p) != NULL);
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,458 @@
|
||||
/*----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2024, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Definition of page queues for each block size
|
||||
----------------------------------------------------------- */
|
||||
|
||||
#ifndef MI_IN_PAGE_C
|
||||
#error "this file should be included from 'page.c'"
|
||||
// include to help an IDE
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc/internal.h"
|
||||
#include "mimalloc/atomic.h"
|
||||
#endif
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Minimal alignment in machine words (i.e. `sizeof(void*)`)
|
||||
----------------------------------------------------------- */
|
||||
|
||||
#if (MI_MAX_ALIGN_SIZE > 4*MI_INTPTR_SIZE)
|
||||
#error "define alignment for more than 4x word size for this platform"
|
||||
#elif (MI_MAX_ALIGN_SIZE > 2*MI_INTPTR_SIZE)
|
||||
#define MI_ALIGN4W // 4 machine words minimal alignment
|
||||
#elif (MI_MAX_ALIGN_SIZE > MI_INTPTR_SIZE)
|
||||
#define MI_ALIGN2W // 2 machine words minimal alignment
|
||||
#else
|
||||
// ok, default alignment is 1 word
|
||||
#endif
|
||||
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Queue query
|
||||
----------------------------------------------------------- */
|
||||
|
||||
|
||||
static inline bool mi_page_queue_is_huge(const mi_page_queue_t* pq) {
|
||||
return (pq->block_size == (MI_LARGE_MAX_OBJ_SIZE+sizeof(uintptr_t)));
|
||||
}
|
||||
|
||||
static inline bool mi_page_queue_is_full(const mi_page_queue_t* pq) {
|
||||
return (pq->block_size == (MI_LARGE_MAX_OBJ_SIZE+(2*sizeof(uintptr_t))));
|
||||
}
|
||||
|
||||
static inline bool mi_page_queue_is_special(const mi_page_queue_t* pq) {
|
||||
return (pq->block_size > MI_LARGE_MAX_OBJ_SIZE);
|
||||
}
|
||||
|
||||
static inline size_t mi_page_queue_count(const mi_page_queue_t* pq) {
|
||||
return pq->count;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Bins
|
||||
----------------------------------------------------------- */
|
||||
|
||||
// Return the bin for a given field size.
|
||||
// Returns MI_BIN_HUGE if the size is too large.
|
||||
// We use `wsize` for the size in "machine word sizes",
|
||||
// i.e. byte size == `wsize*sizeof(void*)`.
|
||||
static mi_decl_noinline size_t mi_bin(size_t size) {
|
||||
size_t wsize = _mi_wsize_from_size(size);
|
||||
#if defined(MI_ALIGN4W)
|
||||
if mi_likely(wsize <= 4) {
|
||||
return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes
|
||||
}
|
||||
#elif defined(MI_ALIGN2W)
|
||||
if mi_likely(wsize <= 8) {
|
||||
return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes
|
||||
}
|
||||
#else
|
||||
if mi_likely(wsize <= 8) {
|
||||
return (wsize == 0 ? 1 : wsize);
|
||||
}
|
||||
#endif
|
||||
else if mi_unlikely(wsize > MI_LARGE_MAX_OBJ_WSIZE) {
|
||||
return MI_BIN_HUGE;
|
||||
}
|
||||
else {
|
||||
#if defined(MI_ALIGN4W)
|
||||
if (wsize <= 16) { wsize = (wsize+3)&~3; } // round to 4x word sizes
|
||||
#endif
|
||||
wsize--;
|
||||
// find the highest bit
|
||||
const size_t b = (MI_SIZE_BITS - 1 - mi_clz(wsize)); // note: wsize != 0
|
||||
// and use the top 3 bits to determine the bin (~12.5% worst internal fragmentation).
|
||||
// - adjust with 3 because we use do not round the first 8 sizes
|
||||
// which each get an exact bin
|
||||
const size_t bin = ((b << 2) + ((wsize >> (b - 2)) & 0x03)) - 3;
|
||||
mi_assert_internal(bin > 0 && bin < MI_BIN_HUGE);
|
||||
return bin;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Queue of pages with free blocks
|
||||
----------------------------------------------------------- */
|
||||
|
||||
size_t _mi_bin(size_t size) {
|
||||
return mi_bin(size);
|
||||
}
|
||||
|
||||
size_t _mi_bin_size(size_t bin) {
|
||||
mi_assert_internal(bin <= MI_BIN_HUGE);
|
||||
return _mi_theap_empty.pages[bin].block_size;
|
||||
}
|
||||
|
||||
// Good size for allocation
|
||||
mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept {
|
||||
if (size <= MI_LARGE_MAX_OBJ_SIZE) {
|
||||
return _mi_bin_size(mi_bin(size + MI_PADDING_SIZE));
|
||||
}
|
||||
else if (size <= MI_MAX_ALLOC_SIZE) {
|
||||
return _mi_align_up(size + MI_PADDING_SIZE,_mi_os_page_size());
|
||||
}
|
||||
else {
|
||||
return size;
|
||||
}
|
||||
}
|
||||
|
||||
#if (MI_DEBUG>1)
|
||||
static bool mi_page_queue_contains(mi_page_queue_t* queue, const mi_page_t* page) {
|
||||
mi_assert_internal(page != NULL);
|
||||
mi_page_t* list = queue->first;
|
||||
while (list != NULL) {
|
||||
mi_assert_internal(list->next == NULL || list->next->prev == list);
|
||||
mi_assert_internal(list->prev == NULL || list->prev->next == list);
|
||||
if (list == page) break;
|
||||
list = list->next;
|
||||
}
|
||||
return (list == page);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#if (MI_DEBUG>1)
|
||||
static bool mi_theap_contains_queue(const mi_theap_t* theap, const mi_page_queue_t* pq) {
|
||||
return (pq >= &theap->pages[0] && pq <= &theap->pages[MI_BIN_FULL]);
|
||||
}
|
||||
#endif
|
||||
|
||||
bool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq) {
|
||||
MI_UNUSED_RELEASE(theap);
|
||||
if (pq==NULL) return false;
|
||||
size_t count = 0; MI_UNUSED_RELEASE(count);
|
||||
mi_page_t* prev = NULL; MI_UNUSED_RELEASE(prev);
|
||||
for (mi_page_t* page = pq->first; page != NULL; page = page->next) {
|
||||
mi_assert_internal(page->prev == prev);
|
||||
if (mi_page_is_in_full(page)) {
|
||||
mi_assert_internal(_mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 2);
|
||||
}
|
||||
else if (mi_page_is_huge(page)) {
|
||||
mi_assert_internal(_mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 1);
|
||||
}
|
||||
else {
|
||||
mi_assert_internal(mi_page_block_size(page) == pq->block_size);
|
||||
}
|
||||
mi_assert_internal(page->theap == theap);
|
||||
if (page->next == NULL) {
|
||||
mi_assert_internal(pq->last == page);
|
||||
}
|
||||
count++;
|
||||
prev = page;
|
||||
}
|
||||
mi_assert_internal(pq->count == count);
|
||||
return true;
|
||||
}
|
||||
|
||||
static size_t mi_page_bin(const mi_page_t* page) {
|
||||
const size_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page))));
|
||||
mi_assert_internal(bin <= MI_BIN_FULL);
|
||||
return bin;
|
||||
}
|
||||
|
||||
// returns the page bin without using MI_BIN_FULL for statistics
|
||||
size_t _mi_page_stats_bin(const mi_page_t* page) {
|
||||
const size_t bin = (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page)));
|
||||
mi_assert_internal(bin <= MI_BIN_HUGE);
|
||||
return bin;
|
||||
}
|
||||
|
||||
static mi_page_queue_t* mi_theap_page_queue_of(mi_theap_t* theap, const mi_page_t* page) {
|
||||
mi_assert_internal(theap!=NULL);
|
||||
const size_t bin = mi_page_bin(page);
|
||||
mi_page_queue_t* pq = &theap->pages[bin];
|
||||
mi_assert_internal((mi_page_block_size(page) == pq->block_size) ||
|
||||
(mi_page_is_huge(page) && mi_page_queue_is_huge(pq)) ||
|
||||
(mi_page_is_in_full(page) && mi_page_queue_is_full(pq)));
|
||||
return pq;
|
||||
}
|
||||
|
||||
static mi_page_queue_t* mi_page_queue_of(const mi_page_t* page) {
|
||||
mi_theap_t* theap = mi_page_theap(page);
|
||||
mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page);
|
||||
mi_assert_expensive(mi_page_queue_contains(pq, page));
|
||||
return pq;
|
||||
}
|
||||
|
||||
// The current small page array is for efficiency and for each
|
||||
// small size (up to 256) it points directly to the page for that
|
||||
// size without having to compute the bin. This means when the
|
||||
// current free page queue is updated for a small bin, we need to update a
|
||||
// range of entries in `_mi_page_small_free`.
|
||||
static inline void mi_theap_queue_first_update(mi_theap_t* theap, const mi_page_queue_t* pq) {
|
||||
mi_assert_internal(mi_theap_contains_queue(theap,pq));
|
||||
const size_t size = pq->block_size;
|
||||
if (size > MI_SMALL_SIZE_MAX) return;
|
||||
|
||||
mi_page_t* page = pq->first;
|
||||
if (pq->first == NULL) page = (mi_page_t*)&_mi_page_empty;
|
||||
|
||||
// find index in the right direct page array
|
||||
const size_t idx = _mi_wsize_from_size(size);
|
||||
mi_page_t** const pages_free = theap->pages_free_direct;
|
||||
if (pages_free[idx] == page) return; // already set
|
||||
|
||||
// find start slot
|
||||
size_t start;
|
||||
if (idx<=1) {
|
||||
start = 0;
|
||||
}
|
||||
else {
|
||||
// find previous size; due to minimal alignment upto 3 previous bins may need to be skipped
|
||||
mi_assert_internal(pq > &theap->pages[0]); // since idx > 1
|
||||
size_t bin = mi_bin(size);
|
||||
const mi_page_queue_t* prev = pq - 1;
|
||||
while( bin == mi_bin(prev->block_size) && prev > &theap->pages[0]) {
|
||||
prev--;
|
||||
}
|
||||
start = 1 + _mi_wsize_from_size(prev->block_size);
|
||||
if (start > idx) start = idx;
|
||||
}
|
||||
|
||||
// set size range to the right page
|
||||
mi_assert(start <= idx);
|
||||
for (size_t sz = start; sz <= idx; sz++) {
|
||||
pages_free[sz] = page;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
static bool mi_page_queue_is_empty(mi_page_queue_t* queue) {
|
||||
return (queue->first == NULL);
|
||||
}
|
||||
*/
|
||||
|
||||
static void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) {
|
||||
mi_assert_internal(page != NULL);
|
||||
mi_assert_expensive(mi_page_queue_contains(queue, page));
|
||||
mi_assert_internal(queue->count >= 1);
|
||||
mi_assert_internal(mi_page_block_size(page) == queue->block_size ||
|
||||
(mi_page_is_huge(page) && mi_page_queue_is_huge(queue)) ||
|
||||
(mi_page_is_in_full(page) && mi_page_queue_is_full(queue)));
|
||||
mi_theap_t* theap = mi_page_theap(page);
|
||||
if (page->prev != NULL) page->prev->next = page->next;
|
||||
if (page->next != NULL) page->next->prev = page->prev;
|
||||
if (page == queue->last) queue->last = page->prev;
|
||||
if (page == queue->first) {
|
||||
queue->first = page->next;
|
||||
// update first
|
||||
mi_assert_internal(mi_theap_contains_queue(theap, queue));
|
||||
mi_theap_queue_first_update(theap,queue);
|
||||
}
|
||||
theap->page_count--;
|
||||
queue->count--;
|
||||
page->next = NULL;
|
||||
page->prev = NULL;
|
||||
mi_page_set_in_full(page,false);
|
||||
}
|
||||
|
||||
|
||||
static void mi_page_queue_push(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) {
|
||||
mi_assert_internal(mi_page_theap(page) == theap);
|
||||
mi_assert_internal(!mi_page_queue_contains(queue, page));
|
||||
#if MI_HUGE_PAGE_ABANDON
|
||||
mi_assert_internal(_mi_page_segment(page)->page_kind != MI_PAGE_HUGE);
|
||||
#endif
|
||||
mi_assert_internal(mi_page_block_size(page) == queue->block_size ||
|
||||
(mi_page_is_huge(page) && mi_page_queue_is_huge(queue)) ||
|
||||
(mi_page_is_in_full(page) && mi_page_queue_is_full(queue)));
|
||||
|
||||
mi_page_set_in_full(page, mi_page_queue_is_full(queue));
|
||||
|
||||
page->next = queue->first;
|
||||
page->prev = NULL;
|
||||
if (queue->first != NULL) {
|
||||
mi_assert_internal(queue->first->prev == NULL);
|
||||
queue->first->prev = page;
|
||||
queue->first = page;
|
||||
}
|
||||
else {
|
||||
queue->first = queue->last = page;
|
||||
}
|
||||
queue->count++;
|
||||
|
||||
// update direct
|
||||
mi_theap_queue_first_update(theap, queue);
|
||||
theap->page_count++;
|
||||
}
|
||||
|
||||
static void mi_page_queue_push_at_end(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) {
|
||||
mi_assert_internal(mi_page_theap(page) == theap);
|
||||
mi_assert_internal(!mi_page_queue_contains(queue, page));
|
||||
|
||||
mi_assert_internal(mi_page_block_size(page) == queue->block_size ||
|
||||
(mi_page_is_huge(page) && mi_page_queue_is_huge(queue)) ||
|
||||
(mi_page_is_in_full(page) && mi_page_queue_is_full(queue)));
|
||||
|
||||
mi_page_set_in_full(page, mi_page_queue_is_full(queue));
|
||||
|
||||
page->prev = queue->last;
|
||||
page->next = NULL;
|
||||
if (queue->last != NULL) {
|
||||
mi_assert_internal(queue->last->next == NULL);
|
||||
queue->last->next = page;
|
||||
queue->last = page;
|
||||
}
|
||||
else {
|
||||
queue->first = queue->last = page;
|
||||
}
|
||||
queue->count++;
|
||||
|
||||
// update direct
|
||||
if (queue->first == page) {
|
||||
mi_theap_queue_first_update(theap, queue);
|
||||
}
|
||||
theap->page_count++;
|
||||
}
|
||||
|
||||
static void mi_page_queue_move_to_front(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) {
|
||||
mi_assert_internal(mi_page_theap(page) == theap);
|
||||
mi_assert_internal(mi_page_queue_contains(queue, page));
|
||||
if (queue->first == page) return;
|
||||
mi_page_queue_remove(queue, page);
|
||||
mi_page_queue_push(theap, queue, page);
|
||||
mi_assert_internal(queue->first == page);
|
||||
}
|
||||
|
||||
static void mi_page_queue_enqueue_from_ex(mi_page_queue_t* to, mi_page_queue_t* from, bool enqueue_at_end, mi_page_t* page) {
|
||||
mi_assert_internal(page != NULL);
|
||||
mi_assert_internal(from->count >= 1);
|
||||
mi_assert_expensive(mi_page_queue_contains(from, page));
|
||||
mi_assert_expensive(!mi_page_queue_contains(to, page));
|
||||
const size_t bsize = mi_page_block_size(page);
|
||||
MI_UNUSED(bsize);
|
||||
mi_assert_internal((bsize == to->block_size && bsize == from->block_size) ||
|
||||
(bsize == to->block_size && mi_page_queue_is_full(from)) ||
|
||||
(bsize == from->block_size && mi_page_queue_is_full(to)) ||
|
||||
(mi_page_is_huge(page) && mi_page_queue_is_huge(to)) ||
|
||||
(mi_page_is_huge(page) && mi_page_queue_is_full(to)));
|
||||
|
||||
mi_theap_t* theap = mi_page_theap(page);
|
||||
|
||||
// delete from `from`
|
||||
if (page->prev != NULL) page->prev->next = page->next;
|
||||
if (page->next != NULL) page->next->prev = page->prev;
|
||||
if (page == from->last) from->last = page->prev;
|
||||
if (page == from->first) {
|
||||
from->first = page->next;
|
||||
// update first
|
||||
mi_assert_internal(mi_theap_contains_queue(theap, from));
|
||||
mi_theap_queue_first_update(theap, from);
|
||||
}
|
||||
from->count--;
|
||||
|
||||
// insert into `to`
|
||||
to->count++;
|
||||
if (enqueue_at_end) {
|
||||
// enqueue at the end
|
||||
page->prev = to->last;
|
||||
page->next = NULL;
|
||||
if (to->last != NULL) {
|
||||
mi_assert_internal(theap == mi_page_theap(to->last));
|
||||
to->last->next = page;
|
||||
to->last = page;
|
||||
}
|
||||
else {
|
||||
to->first = page;
|
||||
to->last = page;
|
||||
mi_theap_queue_first_update(theap, to);
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (to->first != NULL) {
|
||||
// enqueue at 2nd place
|
||||
mi_assert_internal(theap == mi_page_theap(to->first));
|
||||
mi_page_t* next = to->first->next;
|
||||
page->prev = to->first;
|
||||
page->next = next;
|
||||
to->first->next = page;
|
||||
if (next != NULL) {
|
||||
next->prev = page;
|
||||
}
|
||||
else {
|
||||
to->last = page;
|
||||
}
|
||||
}
|
||||
else {
|
||||
// enqueue at the head (singleton list)
|
||||
page->prev = NULL;
|
||||
page->next = NULL;
|
||||
to->first = page;
|
||||
to->last = page;
|
||||
mi_theap_queue_first_update(theap, to);
|
||||
}
|
||||
}
|
||||
|
||||
mi_page_set_in_full(page, mi_page_queue_is_full(to));
|
||||
}
|
||||
|
||||
static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) {
|
||||
mi_page_queue_enqueue_from_ex(to, from, true /* enqueue at the end */, page);
|
||||
}
|
||||
|
||||
static void mi_page_queue_enqueue_from_full(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) {
|
||||
// note: we could insert at the front to increase reuse, but it slows down certain benchmarks (like `alloc-test`)
|
||||
mi_page_queue_enqueue_from_ex(to, from, true /* enqueue at the end of the `to` queue? */, page);
|
||||
}
|
||||
|
||||
// Only called from `mi_theap_absorb`.
|
||||
size_t _mi_page_queue_append(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_queue_t* append) {
|
||||
mi_assert_internal(mi_theap_contains_queue(theap,pq));
|
||||
mi_assert_internal(pq->block_size == append->block_size);
|
||||
|
||||
if (append->first==NULL) return 0;
|
||||
|
||||
// set append pages to new theap and count
|
||||
size_t count = 0;
|
||||
for (mi_page_t* page = append->first; page != NULL; page = page->next) {
|
||||
mi_page_set_theap(page, theap);
|
||||
count++;
|
||||
}
|
||||
mi_assert_internal(count == append->count);
|
||||
|
||||
if (pq->last==NULL) {
|
||||
// take over afresh
|
||||
mi_assert_internal(pq->first==NULL);
|
||||
pq->first = append->first;
|
||||
pq->last = append->last;
|
||||
mi_theap_queue_first_update(theap, pq);
|
||||
}
|
||||
else {
|
||||
// append to end
|
||||
mi_assert_internal(pq->last!=NULL);
|
||||
mi_assert_internal(append->first!=NULL);
|
||||
pq->last->next = append->first;
|
||||
append->first->prev = pq->last;
|
||||
pq->last = append->last;
|
||||
}
|
||||
pq->count += append->count;
|
||||
|
||||
return count;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,261 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2025, Microsoft Research, Daan Leijen, Alon Zakai
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
// This file is included in `src/prim/prim.c`
|
||||
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc/internal.h"
|
||||
#include "mimalloc/atomic.h"
|
||||
#include "mimalloc/prim.h"
|
||||
|
||||
#include <sched.h> // sched_yield
|
||||
#include <unistd.h> // getentropy
|
||||
|
||||
// Design
|
||||
// ======
|
||||
//
|
||||
// mimalloc is built on top of emmalloc. emmalloc is a minimal allocator on top
|
||||
// of sbrk. The reason for having three layers here is that we want mimalloc to
|
||||
// be able to allocate and release system memory properly, the same way it would
|
||||
// when using VirtualAlloc on Windows or mmap on POSIX, and sbrk is too limited.
|
||||
// Specifically, sbrk can only go up and down, and not "skip" over regions, and
|
||||
// so we end up either never freeing memory to the system, or we can get stuck
|
||||
// with holes.
|
||||
//
|
||||
// Atm wasm generally does *not* free memory back the system: once grown, we do
|
||||
// not shrink back down (https://github.com/WebAssembly/design/issues/1397).
|
||||
// However, that is expected to improve
|
||||
// (https://github.com/WebAssembly/memory-control/blob/main/proposals/memory-control/Overview.md)
|
||||
// and so we do not want to bake those limitations in here.
|
||||
//
|
||||
// Even without that issue, we want our system allocator to handle holes, that
|
||||
// is, it should merge freed regions and allow allocating new content there of
|
||||
// the full size, etc., so that we do not waste space. That means that the
|
||||
// system allocator really does need to handle the general problem of allocating
|
||||
// and freeing variable-sized chunks of memory in a random order, like malloc/
|
||||
// free do. And so it makes sense to layer mimalloc on top of such an
|
||||
// implementation.
|
||||
//
|
||||
// emmalloc makes sense for the lower level because it is small and simple while
|
||||
// still fully handling merging of holes etc. It is not the most efficient
|
||||
// allocator, but our assumption is that mimalloc needs to be fast while the
|
||||
// system allocator underneath it is called much less frequently.
|
||||
//
|
||||
|
||||
//---------------------------------------------
|
||||
// init
|
||||
//---------------------------------------------
|
||||
|
||||
void _mi_prim_mem_init( mi_os_mem_config_t* config) {
|
||||
config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB
|
||||
config->alloc_granularity = 16;
|
||||
config->has_overcommit = false;
|
||||
config->has_partial_free = false;
|
||||
config->has_virtual_reserve = false;
|
||||
}
|
||||
|
||||
extern void emmalloc_free(void*);
|
||||
|
||||
int _mi_prim_free(void* addr, size_t size) {
|
||||
if (size==0) return 0;
|
||||
emmalloc_free(addr);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// Allocation
|
||||
//---------------------------------------------
|
||||
|
||||
extern void* emmalloc_memalign(size_t alignment, size_t size);
|
||||
|
||||
// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
|
||||
int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
|
||||
MI_UNUSED(try_alignment); MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr);
|
||||
*is_large = false;
|
||||
// TODO: Track the highest address ever seen; first uses of it are zeroes.
|
||||
// That assumes no one else uses sbrk but us (they could go up,
|
||||
// scribble, and then down), but we could assert on that perhaps.
|
||||
*is_zero = false;
|
||||
// emmalloc has a minimum alignment size.
|
||||
#define MIN_EMMALLOC_ALIGN 8
|
||||
if (try_alignment < MIN_EMMALLOC_ALIGN) {
|
||||
try_alignment = MIN_EMMALLOC_ALIGN;
|
||||
}
|
||||
void* p = emmalloc_memalign(try_alignment, size);
|
||||
*addr = p;
|
||||
if (p == 0) {
|
||||
return ENOMEM;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// Commit/Reset
|
||||
//---------------------------------------------
|
||||
|
||||
int _mi_prim_commit(void* addr, size_t size, bool* is_zero) {
|
||||
MI_UNUSED(addr); MI_UNUSED(size);
|
||||
// See TODO above.
|
||||
*is_zero = false;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) {
|
||||
MI_UNUSED(addr); MI_UNUSED(size);
|
||||
*needs_recommit = false;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int _mi_prim_reset(void* addr, size_t size) {
|
||||
MI_UNUSED(addr); MI_UNUSED(size);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int _mi_prim_reuse(void* addr, size_t size) {
|
||||
MI_UNUSED(addr); MI_UNUSED(size);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int _mi_prim_protect(void* addr, size_t size, bool protect) {
|
||||
MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(protect);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// Huge pages and NUMA nodes
|
||||
//---------------------------------------------
|
||||
|
||||
int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {
|
||||
MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node);
|
||||
*is_zero = true;
|
||||
*addr = NULL;
|
||||
return ENOSYS;
|
||||
}
|
||||
|
||||
size_t _mi_prim_numa_node(void) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t _mi_prim_numa_node_count(void) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
//----------------------------------------------------------------
|
||||
// Clock
|
||||
//----------------------------------------------------------------
|
||||
|
||||
#include <emscripten/html5.h>
|
||||
|
||||
mi_msecs_t _mi_prim_clock_now(void) {
|
||||
return emscripten_date_now();
|
||||
}
|
||||
|
||||
|
||||
//----------------------------------------------------------------
|
||||
// Process info
|
||||
//----------------------------------------------------------------
|
||||
|
||||
void _mi_prim_process_info(mi_process_info_t* pinfo)
|
||||
{
|
||||
// use defaults
|
||||
MI_UNUSED(pinfo);
|
||||
}
|
||||
|
||||
|
||||
//----------------------------------------------------------------
|
||||
// Output
|
||||
//----------------------------------------------------------------
|
||||
|
||||
#include <emscripten/console.h>
|
||||
|
||||
void _mi_prim_out_stderr( const char* msg) {
|
||||
emscripten_console_error(msg);
|
||||
}
|
||||
|
||||
|
||||
//----------------------------------------------------------------
|
||||
// Environment
|
||||
//----------------------------------------------------------------
|
||||
|
||||
bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {
|
||||
// For code size reasons, do not support environ customization for now.
|
||||
MI_UNUSED(name);
|
||||
MI_UNUSED(result);
|
||||
MI_UNUSED(result_size);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
//----------------------------------------------------------------
|
||||
// Random
|
||||
//----------------------------------------------------------------
|
||||
|
||||
bool _mi_prim_random_buf(void* buf, size_t buf_len) {
|
||||
int err = getentropy(buf, buf_len);
|
||||
return !err;
|
||||
}
|
||||
|
||||
|
||||
//----------------------------------------------------------------
|
||||
// Thread init/done
|
||||
//----------------------------------------------------------------
|
||||
|
||||
#if defined(MI_USE_PTHREADS)
|
||||
|
||||
// use pthread local storage keys to detect thread ending
|
||||
// (and used with MI_TLS_PTHREADS for the default theap)
|
||||
pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1);
|
||||
|
||||
static void mi_pthread_done(void* value) {
|
||||
if (value!=NULL) {
|
||||
_mi_thread_done((mi_theap_t*)value);
|
||||
}
|
||||
}
|
||||
|
||||
void _mi_prim_thread_init_auto_done(void) {
|
||||
mi_assert_internal(_mi_heap_default_key == (pthread_key_t)(-1));
|
||||
pthread_key_create(&_mi_heap_default_key, &mi_pthread_done);
|
||||
}
|
||||
|
||||
void _mi_prim_thread_done_auto_done(void) {
|
||||
if (_mi_heap_default_key != (pthread_key_t)(-1)) { // do not leak the key, see issue #809
|
||||
pthread_key_delete(_mi_heap_default_key);
|
||||
}
|
||||
}
|
||||
|
||||
void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
|
||||
if (_mi_heap_default_key != (pthread_key_t)(-1)) { // can happen during recursive invocation on freeBSD
|
||||
pthread_setspecific(_mi_heap_default_key, theap);
|
||||
}
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
void _mi_prim_thread_init_auto_done(void) {
|
||||
// nothing
|
||||
}
|
||||
|
||||
void _mi_prim_thread_done_auto_done(void) {
|
||||
// nothing
|
||||
}
|
||||
|
||||
void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
|
||||
MI_UNUSED(theap);
|
||||
}
|
||||
#endif
|
||||
|
||||
bool _mi_prim_thread_is_in_threadpool(void) {
|
||||
return false;
|
||||
}
|
||||
|
||||
void _mi_prim_thread_yield(void) {
|
||||
sched_yield();
|
||||
}
|
||||
@@ -0,0 +1,481 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2022, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc/internal.h"
|
||||
|
||||
#if defined(MI_MALLOC_OVERRIDE)
|
||||
|
||||
#if !defined(__APPLE__)
|
||||
#error "this file should only be included on macOS"
|
||||
#endif
|
||||
|
||||
/* ------------------------------------------------------
|
||||
Override system malloc on macOS
|
||||
This is done through the malloc zone interface.
|
||||
It seems to be most robust in combination with interposing
|
||||
though or otherwise we may get zone errors as there are could
|
||||
be allocations done by the time we take over the
|
||||
zone.
|
||||
------------------------------------------------------ */
|
||||
|
||||
#include <AvailabilityMacros.h>
|
||||
#include <malloc/malloc.h>
|
||||
#include <string.h> // memset
|
||||
#include <stdlib.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6)
|
||||
// only available from OSX 10.6
|
||||
extern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_import));
|
||||
#endif
|
||||
|
||||
/* ------------------------------------------------------
|
||||
malloc zone members
|
||||
------------------------------------------------------ */
|
||||
|
||||
static bool is_mimalloc_zone( malloc_zone_t* zone );
|
||||
|
||||
static size_t zone_size(malloc_zone_t* zone, const void* p) {
|
||||
if (mi_any_heap_contains(p)) {
|
||||
return mi_usable_size(p);
|
||||
}
|
||||
else if (!is_mimalloc_zone(zone)) { // can happen due to interpose
|
||||
return zone->size(zone,p);
|
||||
}
|
||||
else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
static void* zone_malloc(malloc_zone_t* zone, size_t size) {
|
||||
MI_UNUSED(zone);
|
||||
return mi_malloc(size);
|
||||
}
|
||||
|
||||
static void* zone_calloc(malloc_zone_t* zone, size_t count, size_t size) {
|
||||
MI_UNUSED(zone);
|
||||
return mi_calloc(count, size);
|
||||
}
|
||||
|
||||
static void* zone_valloc(malloc_zone_t* zone, size_t size) {
|
||||
MI_UNUSED(zone);
|
||||
return mi_malloc_aligned(size, _mi_os_page_size());
|
||||
}
|
||||
|
||||
static void zone_free(malloc_zone_t* zone, void* p) {
|
||||
if (mi_any_heap_contains(p)) {
|
||||
mi_free(p); // with the page_map and pagemap_commit=1 we can use the regular free
|
||||
}
|
||||
else if (!is_mimalloc_zone(zone)) { // can happen due to interpose
|
||||
zone->free(zone,p);
|
||||
}
|
||||
}
|
||||
|
||||
static void* zone_realloc(malloc_zone_t* zone, void* p, size_t newsize) {
|
||||
if (p == NULL || mi_any_heap_contains(p)) {
|
||||
return mi_realloc(p, newsize);
|
||||
}
|
||||
else if (!is_mimalloc_zone(zone)) { // can happen due to interpose
|
||||
return zone->realloc(zone,p,newsize);
|
||||
}
|
||||
else {
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static void* zone_memalign(malloc_zone_t* zone, size_t alignment, size_t size) {
|
||||
MI_UNUSED(zone);
|
||||
return mi_malloc_aligned(size,alignment);
|
||||
}
|
||||
|
||||
static void zone_destroy(malloc_zone_t* zone) {
|
||||
if (!is_mimalloc_zone(zone)) {
|
||||
zone->destroy(zone);
|
||||
}
|
||||
}
|
||||
|
||||
static unsigned zone_batch_malloc(malloc_zone_t* zone, size_t size, void** ps, unsigned count) {
|
||||
unsigned i;
|
||||
for (i = 0; i < count; i++) {
|
||||
ps[i] = zone_malloc(zone, size);
|
||||
if (ps[i] == NULL) break;
|
||||
}
|
||||
return i;
|
||||
}
|
||||
|
||||
static void zone_batch_free(malloc_zone_t* zone, void** ps, unsigned count) {
|
||||
for(size_t i = 0; i < count; i++) {
|
||||
zone_free(zone, ps[i]);
|
||||
ps[i] = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static size_t zone_pressure_relief(malloc_zone_t* zone, size_t size) {
|
||||
MI_UNUSED(zone); MI_UNUSED(size);
|
||||
mi_collect(false);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void zone_free_definite_size(malloc_zone_t* zone, void* p, size_t size) {
|
||||
MI_UNUSED(size);
|
||||
zone_free(zone,p);
|
||||
}
|
||||
|
||||
static boolean_t zone_claimed_address(malloc_zone_t* zone, void* p) {
|
||||
MI_UNUSED(zone);
|
||||
return mi_is_in_heap_region(p);
|
||||
}
|
||||
|
||||
|
||||
/* ------------------------------------------------------
|
||||
Introspection members
|
||||
------------------------------------------------------ */
|
||||
|
||||
static kern_return_t intro_enumerator(task_t task, void* p,
|
||||
unsigned type_mask, vm_address_t zone_address,
|
||||
memory_reader_t reader,
|
||||
vm_range_recorder_t recorder)
|
||||
{
|
||||
// todo: enumerate all memory
|
||||
MI_UNUSED(task); MI_UNUSED(p); MI_UNUSED(type_mask); MI_UNUSED(zone_address);
|
||||
MI_UNUSED(reader); MI_UNUSED(recorder);
|
||||
return KERN_SUCCESS;
|
||||
}
|
||||
|
||||
static size_t intro_good_size(malloc_zone_t* zone, size_t size) {
|
||||
MI_UNUSED(zone);
|
||||
return mi_good_size(size);
|
||||
}
|
||||
|
||||
static boolean_t intro_check(malloc_zone_t* zone) {
|
||||
MI_UNUSED(zone);
|
||||
return true;
|
||||
}
|
||||
|
||||
static void intro_print(malloc_zone_t* zone, boolean_t verbose) {
|
||||
MI_UNUSED(zone); MI_UNUSED(verbose);
|
||||
mi_stats_print(NULL);
|
||||
}
|
||||
|
||||
static void intro_log(malloc_zone_t* zone, void* p) {
|
||||
MI_UNUSED(zone); MI_UNUSED(p);
|
||||
// todo?
|
||||
}
|
||||
|
||||
static void intro_force_lock(malloc_zone_t* zone) {
|
||||
MI_UNUSED(zone);
|
||||
// todo?
|
||||
}
|
||||
|
||||
static void intro_force_unlock(malloc_zone_t* zone) {
|
||||
MI_UNUSED(zone);
|
||||
// todo?
|
||||
}
|
||||
|
||||
static void intro_statistics(malloc_zone_t* zone, malloc_statistics_t* stats) {
|
||||
MI_UNUSED(zone);
|
||||
// todo...
|
||||
stats->blocks_in_use = 0;
|
||||
stats->size_in_use = 0;
|
||||
stats->max_size_in_use = 0;
|
||||
stats->size_allocated = 0;
|
||||
}
|
||||
|
||||
static boolean_t intro_zone_locked(malloc_zone_t* zone) {
|
||||
MI_UNUSED(zone);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
/* ------------------------------------------------------
|
||||
At process start, override the default allocator
|
||||
------------------------------------------------------ */
|
||||
|
||||
#if defined(__GNUC__) && !defined(__clang__)
|
||||
#pragma GCC diagnostic ignored "-Wmissing-field-initializers"
|
||||
#endif
|
||||
|
||||
#if defined(__clang__)
|
||||
#pragma clang diagnostic ignored "-Wc99-extensions"
|
||||
#endif
|
||||
|
||||
static malloc_introspection_t mi_introspect = {
|
||||
.enumerator = &intro_enumerator,
|
||||
.good_size = &intro_good_size,
|
||||
.check = &intro_check,
|
||||
.print = &intro_print,
|
||||
.log = &intro_log,
|
||||
.force_lock = &intro_force_lock,
|
||||
.force_unlock = &intro_force_unlock,
|
||||
#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__)
|
||||
.statistics = &intro_statistics,
|
||||
.zone_locked = &intro_zone_locked,
|
||||
#endif
|
||||
};
|
||||
|
||||
static malloc_zone_t mi_malloc_zone = {
|
||||
// note: even with designators, the order is important for C++ compilation
|
||||
//.reserved1 = NULL,
|
||||
//.reserved2 = NULL,
|
||||
.size = &zone_size,
|
||||
.malloc = &zone_malloc,
|
||||
.calloc = &zone_calloc,
|
||||
.valloc = &zone_valloc,
|
||||
.free = &zone_free,
|
||||
.realloc = &zone_realloc,
|
||||
.destroy = &zone_destroy,
|
||||
.zone_name = "mimalloc",
|
||||
.batch_malloc = &zone_batch_malloc,
|
||||
.batch_free = &zone_batch_free,
|
||||
.introspect = &mi_introspect,
|
||||
#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__)
|
||||
#if defined(MAC_OS_X_VERSION_10_14) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14)
|
||||
.version = 10,
|
||||
#else
|
||||
.version = 9,
|
||||
#endif
|
||||
// switch to version 9+ on OSX 10.6 to support memalign.
|
||||
.memalign = &zone_memalign,
|
||||
.free_definite_size = &zone_free_definite_size,
|
||||
#if defined(MAC_OS_X_VERSION_10_7) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7)
|
||||
.pressure_relief = &zone_pressure_relief,
|
||||
#endif
|
||||
#if defined(MAC_OS_X_VERSION_10_14) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14)
|
||||
.claimed_address = &zone_claimed_address,
|
||||
#endif
|
||||
#else
|
||||
.version = 4,
|
||||
#endif
|
||||
};
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
static bool is_mimalloc_zone( malloc_zone_t* zone ) {
|
||||
return (zone==NULL || zone==&mi_malloc_zone);
|
||||
}
|
||||
|
||||
#if defined(MI_OSX_INTERPOSE) && defined(MI_SHARED_LIB_EXPORT)
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Override malloc_xxx and malloc_zone_xxx api's to use only
|
||||
// our mimalloc zone. Since even the loader uses malloc
|
||||
// on macOS, this ensures that all allocations go through
|
||||
// mimalloc (as all calls are interposed).
|
||||
// The main `malloc`, `free`, etc calls are interposed in `alloc-override.c`,
|
||||
// Here, we also override macOS specific API's like
|
||||
// `malloc_zone_calloc` etc. see <https://github.com/aosm/libmalloc/blob/master/man/malloc_zone_malloc.3>
|
||||
// ------------------------------------------------------
|
||||
|
||||
static inline malloc_zone_t* mi_get_default_zone(void) {
|
||||
mi_atomic_do_once {
|
||||
malloc_zone_register(&mi_malloc_zone); // by calling register we avoid a zone error on free (see <http://eatmyrandom.blogspot.com/2010/03/mallocfree-interception-on-mac-os-x.html>)
|
||||
}
|
||||
return &mi_malloc_zone;
|
||||
}
|
||||
|
||||
mi_decl_externc int malloc_jumpstart(uintptr_t cookie);
|
||||
mi_decl_externc void _malloc_fork_prepare(void);
|
||||
mi_decl_externc void _malloc_fork_parent(void);
|
||||
mi_decl_externc void _malloc_fork_child(void);
|
||||
|
||||
|
||||
static malloc_zone_t* mi_malloc_create_zone(vm_size_t size, unsigned flags) {
|
||||
MI_UNUSED(size); MI_UNUSED(flags);
|
||||
return mi_get_default_zone();
|
||||
}
|
||||
|
||||
static malloc_zone_t* mi_malloc_default_zone (void) {
|
||||
return mi_get_default_zone();
|
||||
}
|
||||
|
||||
static malloc_zone_t* mi_malloc_default_purgeable_zone(void) {
|
||||
return mi_get_default_zone();
|
||||
}
|
||||
|
||||
static void mi_malloc_destroy_zone(malloc_zone_t* zone) {
|
||||
MI_UNUSED(zone);
|
||||
// nothing.
|
||||
}
|
||||
|
||||
static kern_return_t mi_malloc_get_all_zones (task_t task, memory_reader_t mr, vm_address_t** addresses, unsigned* count) {
|
||||
MI_UNUSED(task); MI_UNUSED(mr);
|
||||
if (addresses != NULL) *addresses = NULL;
|
||||
if (count != NULL) *count = 0;
|
||||
return KERN_SUCCESS;
|
||||
}
|
||||
|
||||
static const char* mi_malloc_get_zone_name(malloc_zone_t* zone) {
|
||||
return (zone == NULL ? mi_malloc_zone.zone_name : zone->zone_name);
|
||||
}
|
||||
|
||||
static void mi_malloc_set_zone_name(malloc_zone_t* zone, const char* name) {
|
||||
MI_UNUSED(zone); MI_UNUSED(name);
|
||||
}
|
||||
|
||||
static int mi_malloc_jumpstart(uintptr_t cookie) {
|
||||
MI_UNUSED(cookie);
|
||||
return 1; // or 0 for no error?
|
||||
}
|
||||
|
||||
static void mi__malloc_fork_prepare(void) {
|
||||
// nothing
|
||||
}
|
||||
static void mi__malloc_fork_parent(void) {
|
||||
// nothing
|
||||
}
|
||||
static void mi__malloc_fork_child(void) {
|
||||
// nothing
|
||||
}
|
||||
|
||||
static void mi_malloc_printf(const char* fmt, ...) {
|
||||
MI_UNUSED(fmt);
|
||||
}
|
||||
|
||||
static bool zone_check(malloc_zone_t* zone) {
|
||||
MI_UNUSED(zone);
|
||||
return true;
|
||||
}
|
||||
|
||||
static malloc_zone_t* zone_from_ptr(const void* p) {
|
||||
MI_UNUSED(p);
|
||||
return (mi_any_heap_contains(p) ? mi_get_default_zone() : NULL);
|
||||
}
|
||||
|
||||
static void zone_log(malloc_zone_t* zone, void* p) {
|
||||
MI_UNUSED(zone); MI_UNUSED(p);
|
||||
}
|
||||
|
||||
static void zone_print(malloc_zone_t* zone, bool b) {
|
||||
MI_UNUSED(zone); MI_UNUSED(b);
|
||||
}
|
||||
|
||||
static void zone_print_ptr_info(void* p) {
|
||||
MI_UNUSED(p);
|
||||
}
|
||||
|
||||
static void zone_register(malloc_zone_t* zone) {
|
||||
MI_UNUSED(zone);
|
||||
}
|
||||
|
||||
static void zone_unregister(malloc_zone_t* zone) {
|
||||
MI_UNUSED(zone);
|
||||
}
|
||||
|
||||
// use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1`
|
||||
// See: <https://books.google.com/books?id=K8vUkpOXhN4C&pg=PA73>
|
||||
struct mi_interpose_s {
|
||||
const void* replacement;
|
||||
const void* target;
|
||||
};
|
||||
#define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun }
|
||||
#define MI_INTERPOSE_MI(fun) MI_INTERPOSE_FUN(fun,mi_##fun)
|
||||
#define MI_INTERPOSE_ZONE(fun) MI_INTERPOSE_FUN(malloc_##fun,fun)
|
||||
__attribute__((used)) static const struct mi_interpose_s _mi_zone_interposes[] __attribute__((section("__DATA, __interpose"))) =
|
||||
{
|
||||
|
||||
MI_INTERPOSE_MI(malloc_create_zone),
|
||||
MI_INTERPOSE_MI(malloc_default_purgeable_zone),
|
||||
MI_INTERPOSE_MI(malloc_default_zone),
|
||||
MI_INTERPOSE_MI(malloc_destroy_zone),
|
||||
MI_INTERPOSE_MI(malloc_get_all_zones),
|
||||
MI_INTERPOSE_MI(malloc_get_zone_name),
|
||||
MI_INTERPOSE_MI(malloc_jumpstart),
|
||||
MI_INTERPOSE_MI(malloc_printf),
|
||||
MI_INTERPOSE_MI(malloc_set_zone_name),
|
||||
MI_INTERPOSE_MI(_malloc_fork_child),
|
||||
MI_INTERPOSE_MI(_malloc_fork_parent),
|
||||
MI_INTERPOSE_MI(_malloc_fork_prepare),
|
||||
|
||||
MI_INTERPOSE_ZONE(zone_batch_free),
|
||||
MI_INTERPOSE_ZONE(zone_batch_malloc),
|
||||
MI_INTERPOSE_ZONE(zone_calloc),
|
||||
MI_INTERPOSE_ZONE(zone_check),
|
||||
MI_INTERPOSE_ZONE(zone_free),
|
||||
MI_INTERPOSE_ZONE(zone_from_ptr),
|
||||
MI_INTERPOSE_ZONE(zone_log),
|
||||
MI_INTERPOSE_ZONE(zone_malloc),
|
||||
MI_INTERPOSE_ZONE(zone_memalign),
|
||||
MI_INTERPOSE_ZONE(zone_print),
|
||||
MI_INTERPOSE_ZONE(zone_print_ptr_info),
|
||||
MI_INTERPOSE_ZONE(zone_realloc),
|
||||
MI_INTERPOSE_ZONE(zone_register),
|
||||
MI_INTERPOSE_ZONE(zone_unregister),
|
||||
MI_INTERPOSE_ZONE(zone_valloc)
|
||||
};
|
||||
|
||||
|
||||
#else
|
||||
|
||||
// ------------------------------------------------------
|
||||
// hook into the zone api's without interposing
|
||||
// This is the official way of adding an allocator but
|
||||
// it seems less robust than using interpose.
|
||||
// ------------------------------------------------------
|
||||
|
||||
static inline malloc_zone_t* mi_get_default_zone(void)
|
||||
{
|
||||
// The first returned zone is the real default
|
||||
malloc_zone_t** zones = NULL;
|
||||
unsigned count = 0;
|
||||
kern_return_t ret = malloc_get_all_zones(0, NULL, (vm_address_t**)&zones, &count);
|
||||
if (ret == KERN_SUCCESS && count > 0) {
|
||||
return zones[0];
|
||||
}
|
||||
else {
|
||||
// fallback
|
||||
return malloc_default_zone();
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(__clang__)
|
||||
__attribute__((constructor(101))) // highest priority
|
||||
#else
|
||||
__attribute__((constructor)) // priority level is not supported by gcc
|
||||
#endif
|
||||
__attribute__((used))
|
||||
static void _mi_macos_override_malloc(void) {
|
||||
malloc_zone_t* purgeable_zone = NULL;
|
||||
|
||||
#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6)
|
||||
// force the purgeable zone to exist to avoid strange bugs
|
||||
if (malloc_default_purgeable_zone) {
|
||||
purgeable_zone = malloc_default_purgeable_zone();
|
||||
}
|
||||
#endif
|
||||
|
||||
// Register our zone.
|
||||
// thomcc: I think this is still needed to put us in the zone list.
|
||||
malloc_zone_register(&mi_malloc_zone);
|
||||
// Unregister the default zone, this makes our zone the new default
|
||||
// as that was the last registered.
|
||||
malloc_zone_t *default_zone = mi_get_default_zone();
|
||||
// thomcc: Unsure if the next test is *always* false or just false in the
|
||||
// cases I've tried. I'm also unsure if the code inside is needed. at all
|
||||
if (default_zone != &mi_malloc_zone) {
|
||||
malloc_zone_unregister(default_zone);
|
||||
|
||||
// Reregister the default zone so free and realloc in that zone keep working.
|
||||
malloc_zone_register(default_zone);
|
||||
}
|
||||
|
||||
// Unregister, and re-register the purgeable_zone to avoid bugs if it occurs
|
||||
// earlier than the default zone.
|
||||
if (purgeable_zone != NULL) {
|
||||
malloc_zone_unregister(purgeable_zone);
|
||||
malloc_zone_register(purgeable_zone);
|
||||
}
|
||||
|
||||
}
|
||||
#endif // MI_OSX_INTERPOSE
|
||||
|
||||
#endif // MI_MALLOC_OVERRIDE
|
||||
@@ -0,0 +1,9 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
// We use the unix/prim.c with the mmap API on macOSX
|
||||
#include "../unix/prim.c"
|
||||
@@ -0,0 +1,76 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
// Select the implementation of the primitives
|
||||
// depending on the OS.
|
||||
|
||||
#if defined(_WIN32)
|
||||
#include "windows/prim.c" // VirtualAlloc (Windows)
|
||||
|
||||
#elif defined(__APPLE__)
|
||||
#include "osx/prim.c" // macOSX (actually defers to mmap in unix/prim.c)
|
||||
|
||||
#elif defined(__wasi__)
|
||||
#define MI_USE_SBRK
|
||||
#include "wasi/prim.c" // memory-grow or sbrk (Wasm)
|
||||
|
||||
#elif defined(__EMSCRIPTEN__)
|
||||
#include "emscripten/prim.c" // emmalloc_*, + pthread support
|
||||
|
||||
#else
|
||||
#include "unix/prim.c" // mmap() (Linux, macOSX, BSD, Illumnos, Haiku, DragonFly, etc.)
|
||||
|
||||
#endif
|
||||
|
||||
// Generic process initialization
|
||||
#ifndef MI_PRIM_HAS_PROCESS_ATTACH
|
||||
#if defined(__GNUC__) || defined(__clang__)
|
||||
// gcc,clang: use the constructor/destructor attribute
|
||||
// which for both seem to run before regular constructors/destructors
|
||||
#if defined(__clang__)
|
||||
#define mi_attr_constructor __attribute__((constructor(101)))
|
||||
#define mi_attr_destructor __attribute__((destructor(101)))
|
||||
#else
|
||||
#define mi_attr_constructor __attribute__((constructor))
|
||||
#define mi_attr_destructor __attribute__((destructor))
|
||||
#endif
|
||||
static void mi_attr_constructor mi_process_attach(void) {
|
||||
_mi_auto_process_init();
|
||||
}
|
||||
static void mi_attr_destructor mi_process_detach(void) {
|
||||
_mi_auto_process_done();
|
||||
}
|
||||
#elif defined(__cplusplus)
|
||||
// C++: use static initialization to detect process start/end
|
||||
// This is not guaranteed to be first/last but the best we can generally do?
|
||||
struct mi_init_done_t {
|
||||
mi_init_done_t() {
|
||||
_mi_auto_process_init();
|
||||
}
|
||||
~mi_init_done_t() {
|
||||
_mi_auto_process_done();
|
||||
}
|
||||
};
|
||||
static mi_init_done_t mi_init_done;
|
||||
#else
|
||||
#pragma message("define a way to call _mi_auto_process_init/done on your platform")
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// Generic allocator init/done callback
|
||||
#ifndef MI_PRIM_HAS_ALLOCATOR_INIT
|
||||
bool _mi_is_redirected(void) {
|
||||
return false;
|
||||
}
|
||||
bool _mi_allocator_init(const char** message) {
|
||||
if (message != NULL) { *message = NULL; }
|
||||
return true;
|
||||
}
|
||||
void _mi_allocator_done(void) {
|
||||
// nothing to do
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,9 @@
|
||||
## Portability Primitives
|
||||
|
||||
This is the portability layer where all primitives needed from the OS are defined.
|
||||
|
||||
- `include/mimalloc/prim.h`: primitive portability API definition.
|
||||
- `prim.c`: Selects one of `unix/prim.c`, `wasi/prim.c`, or `windows/prim.c` depending on the host platform
|
||||
(and on macOS, `osx/prim.c` defers to `unix/prim.c`).
|
||||
|
||||
Note: still work in progress, there may still be places in the sources that still depend on OS ifdef's.
|
||||
@@ -0,0 +1,995 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
// This file is included in `src/prim/prim.c`
|
||||
|
||||
#ifndef _DEFAULT_SOURCE
|
||||
#define _DEFAULT_SOURCE // ensure mmap flags and syscall are defined
|
||||
#endif
|
||||
|
||||
#if defined(__sun)
|
||||
// illumos provides new mman.h api when any of these are defined
|
||||
// otherwise the old api based on caddr_t which predates the void pointers one.
|
||||
// stock solaris provides only the former, chose to atomically to discard those
|
||||
// flags only here rather than project wide tough.
|
||||
#undef _XOPEN_SOURCE
|
||||
#undef _POSIX_C_SOURCE
|
||||
#endif
|
||||
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc/internal.h"
|
||||
#include "mimalloc/prim.h"
|
||||
|
||||
#include <sys/mman.h> // mmap
|
||||
#include <unistd.h> // sysconf, sleep
|
||||
#include <fcntl.h> // open, close, read, access
|
||||
#include <stdlib.h> // getenv, arc4random_buf
|
||||
|
||||
#if defined(__linux__)
|
||||
#include <features.h>
|
||||
#include <sys/prctl.h> // THP disable, PR_SET_VMA
|
||||
#include <sys/sysinfo.h> // sysinfo
|
||||
#if defined(__GLIBC__) && !defined(PR_SET_VMA)
|
||||
#include <linux/prctl.h>
|
||||
#endif
|
||||
#if defined(__GLIBC__)
|
||||
#include <linux/mman.h> // linux mmap flags
|
||||
#else
|
||||
#include <sys/mman.h>
|
||||
#endif
|
||||
#elif defined(__APPLE__)
|
||||
#include <AvailabilityMacros.h>
|
||||
#include <TargetConditionals.h>
|
||||
#if !defined(TARGET_OS_OSX) || TARGET_OS_OSX // see issue #879, used to be (!TARGET_IOS_IPHONE && !TARGET_IOS_SIMULATOR)
|
||||
#include <mach/vm_statistics.h> // VM_MAKE_TAG, VM_FLAGS_SUPERPAGE_SIZE_2MB, etc.
|
||||
#endif
|
||||
#if !defined(MAC_OS_X_VERSION_10_7)
|
||||
#define MAC_OS_X_VERSION_10_7 1070
|
||||
#endif
|
||||
#include <sys/sysctl.h>
|
||||
#elif defined(__FreeBSD__) || defined(__DragonFly__)
|
||||
#include <sys/param.h>
|
||||
#if __FreeBSD_version >= 1200000
|
||||
#include <sys/cpuset.h>
|
||||
#include <sys/domainset.h>
|
||||
#endif
|
||||
#include <sys/sysctl.h>
|
||||
#endif
|
||||
|
||||
#if (defined(__linux__) && !defined(__ANDROID__)) || defined(__FreeBSD__)
|
||||
#define MI_HAS_SYSCALL_H
|
||||
#include <sys/syscall.h>
|
||||
#endif
|
||||
|
||||
#if !defined(MADV_DONTNEED) && defined(POSIX_MADV_DONTNEED) // QNX
|
||||
#define MADV_DONTNEED POSIX_MADV_DONTNEED
|
||||
#endif
|
||||
#if !defined(MADV_FREE) && defined(POSIX_MADV_FREE) // QNX
|
||||
#define MADV_FREE POSIX_MADV_FREE
|
||||
#endif
|
||||
|
||||
#define MI_UNIX_LARGE_PAGE_SIZE (2*MI_MiB) // TODO: can we query the OS for this?
|
||||
|
||||
//------------------------------------------------------------------------------------
|
||||
// Use syscalls for some primitives to allow for libraries that override open/read/close etc.
|
||||
// and do allocation themselves; using syscalls prevents recursion when mimalloc is
|
||||
// still initializing (issue #713)
|
||||
// Declare inline to avoid unused function warnings.
|
||||
//------------------------------------------------------------------------------------
|
||||
|
||||
#if defined(MI_HAS_SYSCALL_H) && defined(SYS_open) && defined(SYS_close) && defined(SYS_read) && defined(SYS_access)
|
||||
|
||||
static inline int mi_prim_open(const char* fpath, int open_flags) {
|
||||
return syscall(SYS_open,fpath,open_flags,0);
|
||||
}
|
||||
static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {
|
||||
return syscall(SYS_read,fd,buf,bufsize);
|
||||
}
|
||||
static inline int mi_prim_close(int fd) {
|
||||
return syscall(SYS_close,fd);
|
||||
}
|
||||
static inline int mi_prim_access(const char *fpath, int mode) {
|
||||
return syscall(SYS_access,fpath,mode);
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
static inline int mi_prim_open(const char* fpath, int open_flags) {
|
||||
return open(fpath,open_flags);
|
||||
}
|
||||
static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {
|
||||
return read(fd,buf,bufsize);
|
||||
}
|
||||
static inline int mi_prim_close(int fd) {
|
||||
return close(fd);
|
||||
}
|
||||
static inline int mi_prim_access(const char *fpath, int mode) {
|
||||
return access(fpath,mode);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// init
|
||||
//---------------------------------------------
|
||||
|
||||
static bool unix_detect_overcommit(void) {
|
||||
bool os_overcommit = true;
|
||||
#if defined(__linux__)
|
||||
int fd = mi_prim_open("/proc/sys/vm/overcommit_memory", O_RDONLY);
|
||||
if (fd >= 0) {
|
||||
char buf[32];
|
||||
ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf));
|
||||
mi_prim_close(fd);
|
||||
// <https://www.kernel.org/doc/Documentation/vm/overcommit-accounting>
|
||||
// 0: heuristic overcommit, 1: always overcommit, 2: never overcommit (ignore NORESERVE)
|
||||
if (nread >= 1) {
|
||||
os_overcommit = (buf[0] == '0' || buf[0] == '1');
|
||||
}
|
||||
}
|
||||
#elif defined(__FreeBSD__)
|
||||
int val = 0;
|
||||
size_t olen = sizeof(val);
|
||||
if (sysctlbyname("vm.overcommit", &val, &olen, NULL, 0) == 0) {
|
||||
os_overcommit = (val != 0);
|
||||
}
|
||||
#else
|
||||
// default: overcommit is true
|
||||
#endif
|
||||
return os_overcommit;
|
||||
}
|
||||
|
||||
static bool unix_detect_thp(void) {
|
||||
bool thp_enabled = false;
|
||||
#if defined(__linux__)
|
||||
int fd = mi_prim_open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
|
||||
if (fd >= 0) {
|
||||
char buf[32];
|
||||
ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf));
|
||||
mi_prim_close(fd);
|
||||
// <https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html>
|
||||
// between brackets is the current value, for example: always [madvise] never
|
||||
if (nread >= 1) {
|
||||
thp_enabled = (_mi_strnstr(buf,32,"[never]") == NULL);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
return thp_enabled;
|
||||
}
|
||||
|
||||
// try to detect the physical memory dynamically (if possible)
|
||||
static void unix_detect_physical_memory( size_t page_size, size_t* physical_memory_in_kib ) {
|
||||
#if defined(CTL_HW) && (defined(HW_PHYSMEM64) || defined(HW_MEMSIZE)) // freeBSD, macOS
|
||||
MI_UNUSED(page_size);
|
||||
int64_t physical_memory = 0;
|
||||
size_t length = sizeof(int64_t);
|
||||
#if defined(HW_PHYSMEM64)
|
||||
int mib[2] = { CTL_HW, HW_PHYSMEM64 };
|
||||
#else
|
||||
int mib[2] = { CTL_HW, HW_MEMSIZE };
|
||||
#endif
|
||||
const int err = sysctl(mib, 2, &physical_memory, &length, NULL, 0);
|
||||
if (err==0 && physical_memory > 0) {
|
||||
const int64_t phys_in_kib = physical_memory / MI_KiB;
|
||||
if (phys_in_kib > 0 && (uint64_t)phys_in_kib <= SIZE_MAX) {
|
||||
*physical_memory_in_kib = (size_t)phys_in_kib;
|
||||
}
|
||||
}
|
||||
#elif defined(__linux__)
|
||||
MI_UNUSED(page_size);
|
||||
struct sysinfo info; _mi_memzero_var(info);
|
||||
const int err = sysinfo(&info);
|
||||
if (err==0 && info.mem_unit > 0 && info.totalram <= SIZE_MAX) {
|
||||
if (info.mem_unit==MI_KiB) {
|
||||
*physical_memory_in_kib = (size_t)info.totalram;
|
||||
}
|
||||
else {
|
||||
size_t total = 0;
|
||||
if (!mi_mul_overflow((size_t)info.totalram, (size_t)info.mem_unit, &total)) {
|
||||
*physical_memory_in_kib = (total / MI_KiB);
|
||||
}
|
||||
}
|
||||
}
|
||||
#elif defined(_SC_PHYS_PAGES) // do not use by default as it might cause allocation (by using `fopen` to parse /proc/meminfo) (issue #1100)
|
||||
const long pphys = sysconf(_SC_PHYS_PAGES);
|
||||
const size_t psize_in_kib = page_size / MI_KiB;
|
||||
if (psize_in_kib > 0 && pphys > 0 && (unsigned long)pphys <= SIZE_MAX && (size_t)pphys <= (SIZE_MAX/psize_in_kib)) {
|
||||
*physical_memory_in_kib = (size_t)pphys * psize_in_kib;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
void _mi_prim_mem_init( mi_os_mem_config_t* config )
|
||||
{
|
||||
long psize = sysconf(_SC_PAGESIZE);
|
||||
if (psize > 0 && (unsigned long)psize < SIZE_MAX) {
|
||||
config->page_size = (size_t)psize;
|
||||
config->alloc_granularity = (size_t)psize;
|
||||
unix_detect_physical_memory(config->page_size, &config->physical_memory_in_kib);
|
||||
}
|
||||
config->large_page_size = MI_UNIX_LARGE_PAGE_SIZE;
|
||||
config->has_overcommit = unix_detect_overcommit();
|
||||
config->has_partial_free = true; // mmap can free in parts
|
||||
config->has_virtual_reserve = true; // todo: check if this true for NetBSD? (for anonymous mmap with PROT_NONE)
|
||||
config->has_transparent_huge_pages = unix_detect_thp();
|
||||
|
||||
// disable transparent huge pages for this process?
|
||||
#if (defined(__linux__) || defined(__ANDROID__)) && defined(PR_GET_THP_DISABLE)
|
||||
if (!mi_option_is_enabled(mi_option_allow_thp)) // disable THP if requested through an option
|
||||
{
|
||||
config->has_transparent_huge_pages = false;
|
||||
if (prctl(PR_GET_THP_DISABLE, 0, 0, 0, 0) == 0) { // -1 on error, 1 if already disabled
|
||||
// Most likely since distros often come with always/madvise settings.
|
||||
// Disabling only for mimalloc process rather than touching system wide settings
|
||||
(void)prctl(PR_SET_THP_DISABLE, 1, 0, 0, 0);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// free
|
||||
//---------------------------------------------
|
||||
|
||||
int _mi_prim_free(void* addr, size_t size ) {
|
||||
if (size==0) return 0;
|
||||
bool err = (munmap(addr, size) == -1);
|
||||
return (err ? errno : 0);
|
||||
}
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// mmap
|
||||
//---------------------------------------------
|
||||
|
||||
// return errno on failure
|
||||
static int unix_madvise(void* addr, size_t size, int advice) {
|
||||
#if defined(__sun)
|
||||
const int res = madvise((caddr_t)addr, size, advice); // Solaris needs cast (issue #520)
|
||||
return (res==0 ? 0 : errno);
|
||||
#elif defined(__QNX__)
|
||||
return posix_madvise(addr, size, advice); // posix returns errno
|
||||
#else
|
||||
const int res = madvise(addr, size, advice); // linux returns -1 on failure and sets errno
|
||||
return (res==0 ? 0 : errno);
|
||||
#endif
|
||||
}
|
||||
|
||||
static void* unix_mmap_prim(void* addr, size_t size, int protect_flags, int flags, int fd) {
|
||||
void* p = mmap(addr, size, protect_flags, flags, fd, 0 /* offset */);
|
||||
#if defined(__linux__) && defined(PR_SET_VMA)
|
||||
if (p!=MAP_FAILED && p!=NULL) {
|
||||
prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, p, size, "mimalloc");
|
||||
}
|
||||
#endif
|
||||
return p;
|
||||
}
|
||||
|
||||
static void* unix_mmap_prim_aligned(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) {
|
||||
MI_UNUSED(try_alignment);
|
||||
void* p = NULL;
|
||||
#if defined(MAP_ALIGNED) // BSD
|
||||
if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) {
|
||||
size_t n = 0;
|
||||
mi_bsr(try_alignment, &n);
|
||||
if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB
|
||||
p = unix_mmap_prim(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd);
|
||||
if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) {
|
||||
int err = errno;
|
||||
_mi_trace_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, addr);
|
||||
}
|
||||
if (p!=MAP_FAILED) return p;
|
||||
// fall back to regular mmap
|
||||
}
|
||||
}
|
||||
#elif defined(MAP_ALIGN) // Solaris
|
||||
if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) {
|
||||
p = unix_mmap_prim((void*)try_alignment, size, protect_flags, flags | MAP_ALIGN, fd); // addr parameter is the required alignment
|
||||
if (p!=MAP_FAILED) return p;
|
||||
// fall back to regular mmap
|
||||
}
|
||||
#endif
|
||||
#if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED)
|
||||
// on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocations
|
||||
if (addr == NULL) {
|
||||
void* hint = _mi_os_get_aligned_hint(try_alignment, size);
|
||||
if (hint != NULL) {
|
||||
p = unix_mmap_prim(hint, size, protect_flags, flags, fd);
|
||||
if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) {
|
||||
#if MI_TRACK_ENABLED // asan sometimes does not instrument errno correctly?
|
||||
int err = 0;
|
||||
#else
|
||||
int err = errno;
|
||||
#endif
|
||||
_mi_trace_message("unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint);
|
||||
}
|
||||
if (p!=MAP_FAILED) return p;
|
||||
// fall back to regular mmap
|
||||
}
|
||||
}
|
||||
#endif
|
||||
// regular mmap
|
||||
p = unix_mmap_prim(addr, size, protect_flags, flags, fd);
|
||||
if (p!=MAP_FAILED) return p;
|
||||
// failed to allocate
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static int unix_mmap_fd(void) {
|
||||
#if defined(VM_MAKE_TAG)
|
||||
// macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99)
|
||||
int os_tag = (int)mi_option_get(mi_option_os_tag);
|
||||
if (os_tag < 100 || os_tag > 255) { os_tag = 254; }
|
||||
return VM_MAKE_TAG(os_tag);
|
||||
#else
|
||||
return -1;
|
||||
#endif
|
||||
}
|
||||
|
||||
static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) {
|
||||
#if !defined(MAP_ANONYMOUS)
|
||||
#define MAP_ANONYMOUS MAP_ANON
|
||||
#endif
|
||||
#if !defined(MAP_NORESERVE)
|
||||
#define MAP_NORESERVE 0
|
||||
#endif
|
||||
void* p = NULL;
|
||||
const int fd = unix_mmap_fd();
|
||||
int flags = MAP_PRIVATE | MAP_ANONYMOUS;
|
||||
if (_mi_os_has_overcommit()) {
|
||||
flags |= MAP_NORESERVE;
|
||||
}
|
||||
#if defined(PROT_MAX)
|
||||
protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD
|
||||
#endif
|
||||
// huge page allocation
|
||||
if (allow_large && (large_only || (_mi_os_canuse_large_page(size, try_alignment) && mi_option_is_enabled(mi_option_allow_large_os_pages)))) {
|
||||
static _Atomic(size_t) large_page_try_ok; // = 0;
|
||||
size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok);
|
||||
if (!large_only && try_ok > 0) {
|
||||
// If the OS is not configured for large OS pages, or the user does not have
|
||||
// enough permission, the `mmap` will always fail (but it might also fail for other reasons).
|
||||
// Therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times
|
||||
// to avoid too many failing calls to mmap.
|
||||
mi_atomic_cas_strong_acq_rel(&large_page_try_ok, &try_ok, try_ok - 1);
|
||||
}
|
||||
else {
|
||||
int lflags = flags & ~MAP_NORESERVE; // using NORESERVE on huge pages seems to fail on Linux
|
||||
int lfd = fd;
|
||||
#ifdef MAP_ALIGNED_SUPER
|
||||
lflags |= MAP_ALIGNED_SUPER;
|
||||
#endif
|
||||
#ifdef MAP_HUGETLB
|
||||
lflags |= MAP_HUGETLB;
|
||||
#endif
|
||||
#ifdef MAP_HUGE_1GB
|
||||
static bool mi_huge_pages_available = true;
|
||||
if (large_only && (size % MI_GiB) == 0 && mi_huge_pages_available) {
|
||||
lflags |= MAP_HUGE_1GB;
|
||||
}
|
||||
else
|
||||
#endif
|
||||
{
|
||||
#ifdef MAP_HUGE_2MB
|
||||
lflags |= MAP_HUGE_2MB;
|
||||
#endif
|
||||
}
|
||||
#ifdef VM_FLAGS_SUPERPAGE_SIZE_2MB
|
||||
lfd |= VM_FLAGS_SUPERPAGE_SIZE_2MB;
|
||||
#endif
|
||||
if (large_only || lflags != flags) {
|
||||
// try large OS page allocation
|
||||
*is_large = true;
|
||||
p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, lflags, lfd);
|
||||
#ifdef MAP_HUGE_1GB
|
||||
if (p == NULL && (lflags & MAP_HUGE_1GB) == MAP_HUGE_1GB) {
|
||||
mi_huge_pages_available = false; // don't try huge 1GiB pages again
|
||||
if (large_only) {
|
||||
_mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (errno: %i)\n", errno);
|
||||
}
|
||||
lflags = ((lflags & ~MAP_HUGE_1GB) | MAP_HUGE_2MB);
|
||||
p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, lflags, lfd);
|
||||
}
|
||||
#endif
|
||||
if (large_only) return p;
|
||||
if (p == NULL) {
|
||||
mi_atomic_store_release(&large_page_try_ok, (size_t)8); // on error, don't try again for the next N allocations
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// regular allocation
|
||||
if (p == NULL) {
|
||||
*is_large = false;
|
||||
p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, flags, fd);
|
||||
#if !defined(MI_NO_THP)
|
||||
if (p != NULL && allow_large && mi_option_is_enabled(mi_option_allow_thp) && _mi_os_canuse_large_page(size, try_alignment)) {
|
||||
#if defined(MADV_HUGEPAGE)
|
||||
// Many Linux systems don't allow MAP_HUGETLB but they support instead
|
||||
// transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE
|
||||
// though since properly aligned allocations will already use large pages if available
|
||||
// in that case -- in particular for our large regions (in `memory.c`).
|
||||
// However, some systems only allow THP if called with explicit `madvise`, so
|
||||
// when large OS pages are enabled for mimalloc, we call `madvise` anyways.
|
||||
if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) {
|
||||
// *is_large = true; // possibly
|
||||
};
|
||||
#elif defined(__sun)
|
||||
struct memcntl_mha cmd = {0};
|
||||
cmd.mha_pagesize = _mi_os_large_page_size();
|
||||
cmd.mha_cmd = MHA_MAPSIZE_VA;
|
||||
if (memcntl((caddr_t)p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) {
|
||||
// *is_large = true; // possibly
|
||||
}
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
|
||||
int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
|
||||
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
|
||||
mi_assert_internal(commit || !allow_large);
|
||||
mi_assert_internal(try_alignment > 0);
|
||||
*is_zero = true;
|
||||
int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE);
|
||||
*addr = unix_mmap(hint_addr, size, try_alignment, protect_flags, false, allow_large, is_large);
|
||||
return (*addr != NULL ? 0 : errno);
|
||||
}
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// Commit/Reset
|
||||
//---------------------------------------------
|
||||
|
||||
static void unix_mprotect_hint(int err) {
|
||||
#if defined(__linux__) && (MI_SECURE>=5) // guard page around every mimalloc page
|
||||
if (err == ENOMEM) {
|
||||
_mi_warning_message("The next warning may be caused by a low memory map limit.\n"
|
||||
" On Linux this is controlled by the vm.max_map_count -- maybe increase it?\n"
|
||||
" For example: sudo sysctl -w vm.max_map_count=262144\n");
|
||||
}
|
||||
#else
|
||||
MI_UNUSED(err);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
int _mi_prim_commit(void* start, size_t size, bool* is_zero) {
|
||||
// commit: ensure we can access the area
|
||||
// note: we may think that *is_zero can be true since the memory
|
||||
// was either from mmap PROT_NONE, or from decommit MADV_DONTNEED, but
|
||||
// we sometimes call commit on a range with still partially committed
|
||||
// memory and `mprotect` does not zero the range.
|
||||
*is_zero = false;
|
||||
int err = mprotect(start, size, (PROT_READ | PROT_WRITE));
|
||||
if (err != 0) {
|
||||
err = errno;
|
||||
unix_mprotect_hint(err);
|
||||
}
|
||||
return err;
|
||||
}
|
||||
|
||||
int _mi_prim_reuse(void* start, size_t size) {
|
||||
MI_UNUSED(start); MI_UNUSED(size);
|
||||
#if defined(__APPLE__) && defined(MADV_FREE_REUSE)
|
||||
return unix_madvise(start, size, MADV_FREE_REUSE);
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) {
|
||||
int err = 0;
|
||||
#if defined(__APPLE__) && defined(MADV_FREE_REUSABLE)
|
||||
// decommit on macOS: use MADV_FREE_REUSABLE as it does immediate rss accounting (issue #1097)
|
||||
err = unix_madvise(start, size, MADV_FREE_REUSABLE);
|
||||
if (err) { err = unix_madvise(start, size, MADV_DONTNEED); }
|
||||
#else
|
||||
// decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE)
|
||||
err = unix_madvise(start, size, MADV_DONTNEED);
|
||||
#endif
|
||||
#if !MI_DEBUG && MI_SECURE<=2
|
||||
*needs_recommit = false;
|
||||
#else
|
||||
*needs_recommit = true;
|
||||
mprotect(start, size, PROT_NONE);
|
||||
#endif
|
||||
/*
|
||||
// decommit: use mmap with MAP_FIXED and PROT_NONE to discard the existing memory (and reduce rss)
|
||||
*needs_recommit = true;
|
||||
const int fd = unix_mmap_fd();
|
||||
void* p = mmap(start, size, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), fd, 0);
|
||||
if (p != start) { err = errno; }
|
||||
*/
|
||||
return err;
|
||||
}
|
||||
|
||||
int _mi_prim_reset(void* start, size_t size) {
|
||||
int err = 0;
|
||||
|
||||
// on macOS can use MADV_FREE_REUSABLE (but we disable this for now as it seems slower)
|
||||
#if 0 && defined(__APPLE__) && defined(MADV_FREE_REUSABLE)
|
||||
err = unix_madvise(start, size, MADV_FREE_REUSABLE);
|
||||
if (err==0) return 0;
|
||||
// fall through
|
||||
#endif
|
||||
|
||||
#if defined(MADV_FREE)
|
||||
// Otherwise, we try to use `MADV_FREE` as that is the fastest. A drawback though is that it
|
||||
// will not reduce the `rss` stats in tools like `top` even though the memory is available
|
||||
// to other processes. With the default `MIMALLOC_PURGE_DECOMMITS=1` we ensure that by
|
||||
// default `MADV_DONTNEED` is used though.
|
||||
static _Atomic(size_t) advice = MI_ATOMIC_VAR_INIT(MADV_FREE);
|
||||
int oadvice = (int)mi_atomic_load_relaxed(&advice);
|
||||
while ((err = unix_madvise(start, size, oadvice)) != 0 && err == EAGAIN) { /* try again */ };
|
||||
if (err == EINVAL && oadvice == MADV_FREE) {
|
||||
// if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on
|
||||
mi_atomic_store_release(&advice, (size_t)MADV_DONTNEED);
|
||||
err = unix_madvise(start, size, MADV_DONTNEED);
|
||||
}
|
||||
#else
|
||||
err = unix_madvise(start, size, MADV_DONTNEED);
|
||||
#endif
|
||||
return err;
|
||||
}
|
||||
|
||||
int _mi_prim_protect(void* start, size_t size, bool protect) {
|
||||
int err = mprotect(start, size, protect ? PROT_NONE : (PROT_READ | PROT_WRITE));
|
||||
if (err != 0) { err = errno; }
|
||||
unix_mprotect_hint(err);
|
||||
return err;
|
||||
}
|
||||
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// Huge page allocation
|
||||
//---------------------------------------------
|
||||
|
||||
#if (MI_INTPTR_SIZE >= 8) && !defined(__HAIKU__) && !defined(__CYGWIN__)
|
||||
|
||||
#ifndef MPOL_PREFERRED
|
||||
#define MPOL_PREFERRED 1
|
||||
#endif
|
||||
|
||||
#if defined(MI_HAS_SYSCALL_H) && defined(SYS_mbind)
|
||||
static long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) {
|
||||
return syscall(SYS_mbind, start, len, mode, nmask, maxnode, flags);
|
||||
}
|
||||
#else
|
||||
static long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) {
|
||||
MI_UNUSED(start); MI_UNUSED(len); MI_UNUSED(mode); MI_UNUSED(nmask); MI_UNUSED(maxnode); MI_UNUSED(flags);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {
|
||||
bool is_large = true;
|
||||
*is_zero = true;
|
||||
*addr = unix_mmap(hint_addr, size, MI_ARENA_SLICE_ALIGN, PROT_READ | PROT_WRITE, true, true, &is_large);
|
||||
if (*addr != NULL && numa_node >= 0 && numa_node < 8*MI_INTPTR_SIZE) { // at most 64 nodes
|
||||
unsigned long numa_mask = (1UL << numa_node);
|
||||
// TODO: does `mbind` work correctly for huge OS pages? should we
|
||||
// use `set_mempolicy` before calling mmap instead?
|
||||
// see: <https://lkml.org/lkml/2017/2/9/875>
|
||||
long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0);
|
||||
if (err != 0) {
|
||||
err = errno;
|
||||
_mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %d (0x%x))\n", numa_node, err, err);
|
||||
}
|
||||
}
|
||||
return (*addr != NULL ? 0 : errno);
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {
|
||||
MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node);
|
||||
*is_zero = false;
|
||||
*addr = NULL;
|
||||
return ENOMEM;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
//---------------------------------------------
|
||||
// NUMA nodes
|
||||
//---------------------------------------------
|
||||
|
||||
#if defined(__linux__)
|
||||
|
||||
size_t _mi_prim_numa_node(void) {
|
||||
#if defined(MI_HAS_SYSCALL_H) && defined(SYS_getcpu)
|
||||
unsigned long node = 0;
|
||||
unsigned long ncpu = 0;
|
||||
long err = syscall(SYS_getcpu, &ncpu, &node, NULL);
|
||||
if (err != 0) return 0;
|
||||
return node;
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
size_t _mi_prim_numa_node_count(void) {
|
||||
char buf[128];
|
||||
unsigned node = 0;
|
||||
for(node = 0; node < 256; node++) {
|
||||
// enumerate node entries -- todo: it there a more efficient way to do this? (but ensure there is no allocation)
|
||||
_mi_snprintf(buf, 127, "/sys/devices/system/node/node%u", node + 1);
|
||||
if (mi_prim_access(buf,R_OK) != 0) break;
|
||||
}
|
||||
return (node+1);
|
||||
}
|
||||
|
||||
#elif defined(__FreeBSD__) && __FreeBSD_version >= 1200000
|
||||
|
||||
size_t _mi_prim_numa_node(void) {
|
||||
domainset_t dom;
|
||||
size_t node;
|
||||
int policy;
|
||||
if (cpuset_getdomain(CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, sizeof(dom), &dom, &policy) == -1) return 0ul;
|
||||
for (node = 0; node < MAXMEMDOM; node++) {
|
||||
if (DOMAINSET_ISSET(node, &dom)) return node;
|
||||
}
|
||||
return 0ul;
|
||||
}
|
||||
|
||||
size_t _mi_prim_numa_node_count(void) {
|
||||
size_t ndomains = 0;
|
||||
size_t len = sizeof(ndomains);
|
||||
if (sysctlbyname("vm.ndomains", &ndomains, &len, NULL, 0) == -1) return 0ul;
|
||||
return ndomains;
|
||||
}
|
||||
|
||||
#elif defined(__DragonFly__)
|
||||
|
||||
size_t _mi_prim_numa_node(void) {
|
||||
// TODO: DragonFly does not seem to provide any userland means to get this information.
|
||||
return 0ul;
|
||||
}
|
||||
|
||||
size_t _mi_prim_numa_node_count(void) {
|
||||
size_t ncpus = 0, nvirtcoresperphys = 0;
|
||||
size_t len = sizeof(size_t);
|
||||
if (sysctlbyname("hw.ncpu", &ncpus, &len, NULL, 0) == -1) return 0ul;
|
||||
if (sysctlbyname("hw.cpu_topology_ht_ids", &nvirtcoresperphys, &len, NULL, 0) == -1) return 0ul;
|
||||
return nvirtcoresperphys * ncpus;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
size_t _mi_prim_numa_node(void) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t _mi_prim_numa_node_count(void) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// Clock
|
||||
// ----------------------------------------------------------------
|
||||
|
||||
#include <time.h>
|
||||
|
||||
#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC)
|
||||
|
||||
mi_msecs_t _mi_prim_clock_now(void) {
|
||||
struct timespec t;
|
||||
#ifdef CLOCK_MONOTONIC
|
||||
clock_gettime(CLOCK_MONOTONIC, &t);
|
||||
#else
|
||||
clock_gettime(CLOCK_REALTIME, &t);
|
||||
#endif
|
||||
return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000);
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
// low resolution timer
|
||||
mi_msecs_t _mi_prim_clock_now(void) {
|
||||
#if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0)
|
||||
return (mi_msecs_t)clock();
|
||||
#elif (CLOCKS_PER_SEC < 1000)
|
||||
return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC);
|
||||
#else
|
||||
return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000);
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
|
||||
//----------------------------------------------------------------
|
||||
// Process info
|
||||
//----------------------------------------------------------------
|
||||
|
||||
#if defined(__unix__) || defined(__unix) || defined(unix) || defined(__APPLE__) || defined(__HAIKU__)
|
||||
#include <stdio.h>
|
||||
#include <unistd.h>
|
||||
#include <sys/resource.h>
|
||||
|
||||
#if defined(__APPLE__)
|
||||
#include <mach/mach.h>
|
||||
#endif
|
||||
|
||||
#if defined(__HAIKU__)
|
||||
#include <kernel/OS.h>
|
||||
#endif
|
||||
|
||||
static mi_msecs_t timeval_secs(const struct timeval* tv) {
|
||||
return ((mi_msecs_t)tv->tv_sec * 1000L) + ((mi_msecs_t)tv->tv_usec / 1000L);
|
||||
}
|
||||
|
||||
void _mi_prim_process_info(mi_process_info_t* pinfo)
|
||||
{
|
||||
struct rusage rusage;
|
||||
getrusage(RUSAGE_SELF, &rusage);
|
||||
pinfo->utime = timeval_secs(&rusage.ru_utime);
|
||||
pinfo->stime = timeval_secs(&rusage.ru_stime);
|
||||
#if !defined(__HAIKU__)
|
||||
pinfo->page_faults = rusage.ru_majflt;
|
||||
#endif
|
||||
#if defined(__HAIKU__)
|
||||
// Haiku does not have (yet?) a way to
|
||||
// get these stats per process
|
||||
thread_info tid;
|
||||
area_info mem;
|
||||
ssize_t c;
|
||||
get_thread_info(find_thread(0), &tid);
|
||||
while (get_next_area_info(tid.team, &c, &mem) == B_OK) {
|
||||
pinfo->peak_rss += mem.ram_size;
|
||||
}
|
||||
pinfo->page_faults = 0;
|
||||
#elif defined(__APPLE__)
|
||||
pinfo->peak_rss = rusage.ru_maxrss; // macos reports in bytes
|
||||
#ifdef MACH_TASK_BASIC_INFO
|
||||
struct mach_task_basic_info info;
|
||||
mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT;
|
||||
if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) {
|
||||
pinfo->current_rss = (size_t)info.resident_size;
|
||||
}
|
||||
#else
|
||||
struct task_basic_info info;
|
||||
mach_msg_type_number_t infoCount = TASK_BASIC_INFO_COUNT;
|
||||
if (task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) {
|
||||
pinfo->current_rss = (size_t)info.resident_size;
|
||||
}
|
||||
#endif
|
||||
#else
|
||||
pinfo->peak_rss = rusage.ru_maxrss * 1024; // Linux/BSD report in KiB
|
||||
#endif
|
||||
// use defaults for commit
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
#ifndef __wasi__
|
||||
// WebAssembly instances are not processes
|
||||
#pragma message("define a way to get process info")
|
||||
#endif
|
||||
|
||||
void _mi_prim_process_info(mi_process_info_t* pinfo)
|
||||
{
|
||||
// use defaults
|
||||
MI_UNUSED(pinfo);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
//----------------------------------------------------------------
|
||||
// Output
|
||||
//----------------------------------------------------------------
|
||||
|
||||
void _mi_prim_out_stderr( const char* msg ) {
|
||||
fputs(msg,stderr);
|
||||
}
|
||||
|
||||
|
||||
//----------------------------------------------------------------
|
||||
// Environment
|
||||
//----------------------------------------------------------------
|
||||
|
||||
#if !defined(MI_USE_ENVIRON) || (MI_USE_ENVIRON!=0)
|
||||
// On Posix systemsr use `environ` to access environment variables
|
||||
// even before the C runtime is initialized.
|
||||
#if defined(__APPLE__) && defined(__has_include) && __has_include(<crt_externs.h>)
|
||||
#include <crt_externs.h>
|
||||
static char** mi_get_environ(void) {
|
||||
return (*_NSGetEnviron());
|
||||
}
|
||||
#else
|
||||
extern char** environ;
|
||||
static char** mi_get_environ(void) {
|
||||
return environ;
|
||||
}
|
||||
#endif
|
||||
bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {
|
||||
if (name==NULL) return false;
|
||||
const size_t len = _mi_strlen(name);
|
||||
if (len == 0) return false;
|
||||
char** env = mi_get_environ();
|
||||
if (env == NULL) return false;
|
||||
// compare up to 10000 entries
|
||||
for (int i = 0; i < 10000 && env[i] != NULL; i++) {
|
||||
const char* s = env[i];
|
||||
if (_mi_strnicmp(name, s, len) == 0 && s[len] == '=') { // case insensitive
|
||||
// found it
|
||||
_mi_strlcpy(result, s + len + 1, result_size);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
#else
|
||||
// fallback: use standard C `getenv` but this cannot be used while initializing the C runtime
|
||||
bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {
|
||||
// cannot call getenv() when still initializing the C runtime.
|
||||
if (_mi_preloading()) return false;
|
||||
const char* s = getenv(name);
|
||||
if (s == NULL) {
|
||||
// we check the upper case name too.
|
||||
char buf[64+1];
|
||||
size_t len = _mi_strnlen(name,sizeof(buf)-1);
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
buf[i] = _mi_toupper(name[i]);
|
||||
}
|
||||
buf[len] = 0;
|
||||
s = getenv(buf);
|
||||
}
|
||||
if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return false;
|
||||
_mi_strlcpy(result, s, result_size);
|
||||
return true;
|
||||
}
|
||||
#endif // !MI_USE_ENVIRON
|
||||
|
||||
|
||||
//----------------------------------------------------------------
|
||||
// Random
|
||||
//----------------------------------------------------------------
|
||||
|
||||
#if defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_15) && (MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_15)
|
||||
#include <CommonCrypto/CommonCryptoError.h>
|
||||
#include <CommonCrypto/CommonRandom.h>
|
||||
|
||||
bool _mi_prim_random_buf(void* buf, size_t buf_len) {
|
||||
// We prefer CCRandomGenerateBytes as it returns an error code while arc4random_buf
|
||||
// may fail silently on macOS. See PR #390, and <https://opensource.apple.com/source/Libc/Libc-1439.40.11/gen/FreeBSD/arc4random.c.auto.html>
|
||||
return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess);
|
||||
}
|
||||
|
||||
#elif defined(__ANDROID__) || defined(__DragonFly__) || \
|
||||
defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
|
||||
defined(__sun) || \
|
||||
(defined(__APPLE__) && (MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_7))
|
||||
|
||||
bool _mi_prim_random_buf(void* buf, size_t buf_len) {
|
||||
arc4random_buf(buf, buf_len);
|
||||
return true;
|
||||
}
|
||||
|
||||
#elif defined(__APPLE__) || defined(__linux__) || defined(__HAIKU__) // also for old apple versions < 10.7 (issue #829)
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <errno.h>
|
||||
|
||||
bool _mi_prim_random_buf(void* buf, size_t buf_len) {
|
||||
// Modern Linux provides `getrandom` but different distributions either use `sys/random.h` or `linux/random.h`
|
||||
// and for the latter the actual `getrandom` call is not always defined.
|
||||
// (see <https://stackoverflow.com/questions/45237324/why-doesnt-getrandom-compile>)
|
||||
// We therefore use a syscall directly and fall back dynamically to /dev/urandom when needed.
|
||||
#if defined(MI_HAS_SYSCALL_H) && defined(SYS_getrandom)
|
||||
#ifndef GRND_NONBLOCK
|
||||
#define GRND_NONBLOCK (1)
|
||||
#endif
|
||||
static _Atomic(uintptr_t) no_getrandom; // = 0
|
||||
if (mi_atomic_load_acquire(&no_getrandom)==0) {
|
||||
ssize_t ret = syscall(SYS_getrandom, buf, buf_len, GRND_NONBLOCK);
|
||||
if (ret >= 0) return (buf_len == (size_t)ret);
|
||||
if (errno != ENOSYS) return false;
|
||||
mi_atomic_store_release(&no_getrandom, (uintptr_t)1); // don't call again, and fall back to /dev/urandom
|
||||
}
|
||||
#endif
|
||||
int flags = O_RDONLY;
|
||||
#if defined(O_CLOEXEC)
|
||||
flags |= O_CLOEXEC;
|
||||
#endif
|
||||
int fd = mi_prim_open("/dev/urandom", flags);
|
||||
if (fd < 0) return false;
|
||||
size_t count = 0;
|
||||
while(count < buf_len) {
|
||||
ssize_t ret = mi_prim_read(fd, (char*)buf + count, buf_len - count);
|
||||
if (ret<=0) {
|
||||
if (errno!=EAGAIN && errno!=EINTR) break;
|
||||
}
|
||||
else {
|
||||
count += ret;
|
||||
}
|
||||
}
|
||||
mi_prim_close(fd);
|
||||
return (count==buf_len);
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
bool _mi_prim_random_buf(void* buf, size_t buf_len) {
|
||||
return false;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
//----------------------------------------------------------------
|
||||
// Thread init/done
|
||||
//----------------------------------------------------------------
|
||||
|
||||
#if defined(MI_USE_PTHREADS)
|
||||
|
||||
// use pthread local storage keys to detect thread ending
|
||||
// (and used with MI_TLS_PTHREADS for the default theap)
|
||||
pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1);
|
||||
|
||||
static void mi_pthread_done(void* value) {
|
||||
if (value!=NULL) {
|
||||
_mi_thread_done((mi_theap_t*)value);
|
||||
}
|
||||
}
|
||||
|
||||
void _mi_prim_thread_init_auto_done(void) {
|
||||
mi_assert_internal(_mi_heap_default_key == (pthread_key_t)(-1));
|
||||
pthread_key_create(&_mi_heap_default_key, &mi_pthread_done);
|
||||
}
|
||||
|
||||
void _mi_prim_thread_done_auto_done(void) {
|
||||
if (_mi_heap_default_key != (pthread_key_t)(-1)) { // do not leak the key, see issue #809
|
||||
pthread_key_delete(_mi_heap_default_key);
|
||||
}
|
||||
}
|
||||
|
||||
void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
|
||||
if (_mi_heap_default_key != (pthread_key_t)(-1)) { // can happen during recursive invocation on freeBSD
|
||||
pthread_setspecific(_mi_heap_default_key, theap);
|
||||
}
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
void _mi_prim_thread_init_auto_done(void) {
|
||||
// nothing
|
||||
}
|
||||
|
||||
void _mi_prim_thread_done_auto_done(void) {
|
||||
// nothing
|
||||
}
|
||||
|
||||
void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
|
||||
MI_UNUSED(theap);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
bool _mi_prim_thread_is_in_threadpool(void) {
|
||||
return false;
|
||||
}
|
||||
|
||||
void _mi_prim_thread_yield(void) {
|
||||
sleep(0);
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
// This file is included in `src/prim/prim.c`
|
||||
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc/internal.h"
|
||||
#include "mimalloc/prim.h"
|
||||
|
||||
#include <stdio.h> // fputs
|
||||
#include <stdlib.h> // getenv
|
||||
#include <unistd.h> // sbrk, sleep
|
||||
|
||||
//---------------------------------------------
|
||||
// Initialize
|
||||
//---------------------------------------------
|
||||
|
||||
void _mi_prim_mem_init( mi_os_mem_config_t* config ) {
|
||||
config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB
|
||||
config->alloc_granularity = 16;
|
||||
config->has_overcommit = false;
|
||||
config->has_partial_free = false;
|
||||
config->has_virtual_reserve = false;
|
||||
}
|
||||
|
||||
//---------------------------------------------
|
||||
// Free
|
||||
//---------------------------------------------
|
||||
|
||||
int _mi_prim_free(void* addr, size_t size ) {
|
||||
MI_UNUSED(addr); MI_UNUSED(size);
|
||||
// wasi theap cannot be shrunk
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// Allocation: sbrk or memory_grow
|
||||
//---------------------------------------------
|
||||
|
||||
#if defined(MI_USE_SBRK)
|
||||
static void* mi_memory_grow( size_t size ) {
|
||||
void* p = sbrk(size);
|
||||
if (p == (void*)(-1)) return NULL;
|
||||
#if !defined(__wasi__) // on wasi this is always zero initialized already (?)
|
||||
memset(p,0,size);
|
||||
#endif
|
||||
return p;
|
||||
}
|
||||
#elif defined(__wasi__)
|
||||
static void* mi_memory_grow( size_t size ) {
|
||||
size_t base = (size > 0 ? __builtin_wasm_memory_grow(0,_mi_divide_up(size, _mi_os_page_size()))
|
||||
: __builtin_wasm_memory_size(0));
|
||||
if (base == SIZE_MAX) return NULL;
|
||||
return (void*)(base * _mi_os_page_size());
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(MI_USE_PTHREADS)
|
||||
static pthread_mutex_t mi_theap_grow_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
#endif
|
||||
|
||||
static void* mi_prim_mem_grow(size_t size, size_t try_alignment) {
|
||||
void* p = NULL;
|
||||
if (try_alignment <= 1) {
|
||||
// `sbrk` is not thread safe in general so try to protect it (we could skip this on WASM but leave it in for now)
|
||||
#if defined(MI_USE_PTHREADS)
|
||||
pthread_mutex_lock(&mi_theap_grow_mutex);
|
||||
#endif
|
||||
p = mi_memory_grow(size);
|
||||
#if defined(MI_USE_PTHREADS)
|
||||
pthread_mutex_unlock(&mi_theap_grow_mutex);
|
||||
#endif
|
||||
}
|
||||
else {
|
||||
void* base = NULL;
|
||||
size_t alloc_size = 0;
|
||||
// to allocate aligned use a lock to try to avoid thread interaction
|
||||
// between getting the current size and actual allocation
|
||||
// (also, `sbrk` is not thread safe in general)
|
||||
#if defined(MI_USE_PTHREADS)
|
||||
pthread_mutex_lock(&mi_theap_grow_mutex);
|
||||
#endif
|
||||
{
|
||||
void* current = mi_memory_grow(0); // get current size
|
||||
if (current != NULL) {
|
||||
void* aligned_current = _mi_align_up_ptr(current, try_alignment); // and align from there to minimize wasted space
|
||||
alloc_size = _mi_align_up( ((uint8_t*)aligned_current - (uint8_t*)current) + size, _mi_os_page_size());
|
||||
base = mi_memory_grow(alloc_size);
|
||||
}
|
||||
}
|
||||
#if defined(MI_USE_PTHREADS)
|
||||
pthread_mutex_unlock(&mi_theap_grow_mutex);
|
||||
#endif
|
||||
if (base != NULL) {
|
||||
p = _mi_align_up_ptr(base, try_alignment);
|
||||
if ((uint8_t*)p + size > (uint8_t*)base + alloc_size) {
|
||||
// another thread used wasm_memory_grow/sbrk in-between and we do not have enough
|
||||
// space after alignment. Give up (and waste the space as we cannot shrink :-( )
|
||||
// (in `mi_os_mem_alloc_aligned` this will fall back to overallocation to align)
|
||||
p = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
/*
|
||||
if (p == NULL) {
|
||||
_mi_warning_message("unable to allocate sbrk/wasm_memory_grow OS memory (%zu bytes, %zu alignment)\n", size, try_alignment);
|
||||
errno = ENOMEM;
|
||||
return NULL;
|
||||
}
|
||||
*/
|
||||
mi_assert_internal( p == NULL || try_alignment == 0 || (uintptr_t)p % try_alignment == 0 );
|
||||
return p;
|
||||
}
|
||||
|
||||
// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
|
||||
int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
|
||||
MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr);
|
||||
*is_large = false;
|
||||
*is_zero = false;
|
||||
*addr = mi_prim_mem_grow(size, try_alignment);
|
||||
return (*addr != NULL ? 0 : ENOMEM);
|
||||
}
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// Commit/Reset/Protect
|
||||
//---------------------------------------------
|
||||
|
||||
int _mi_prim_commit(void* addr, size_t size, bool* is_zero) {
|
||||
MI_UNUSED(addr); MI_UNUSED(size);
|
||||
*is_zero = false;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) {
|
||||
MI_UNUSED(addr); MI_UNUSED(size);
|
||||
*needs_recommit = false;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int _mi_prim_reset(void* addr, size_t size) {
|
||||
MI_UNUSED(addr); MI_UNUSED(size);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int _mi_prim_reuse(void* addr, size_t size) {
|
||||
MI_UNUSED(addr); MI_UNUSED(size);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int _mi_prim_protect(void* addr, size_t size, bool protect) {
|
||||
MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(protect);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// Huge pages and NUMA nodes
|
||||
//---------------------------------------------
|
||||
|
||||
int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {
|
||||
MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node);
|
||||
*is_zero = true;
|
||||
*addr = NULL;
|
||||
return ENOSYS;
|
||||
}
|
||||
|
||||
size_t _mi_prim_numa_node(void) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t _mi_prim_numa_node_count(void) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
//----------------------------------------------------------------
|
||||
// Clock
|
||||
//----------------------------------------------------------------
|
||||
|
||||
#include <time.h>
|
||||
|
||||
#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC)
|
||||
|
||||
mi_msecs_t _mi_prim_clock_now(void) {
|
||||
struct timespec t;
|
||||
#ifdef CLOCK_MONOTONIC
|
||||
clock_gettime(CLOCK_MONOTONIC, &t);
|
||||
#else
|
||||
clock_gettime(CLOCK_REALTIME, &t);
|
||||
#endif
|
||||
return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000);
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
// low resolution timer
|
||||
mi_msecs_t _mi_prim_clock_now(void) {
|
||||
#if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0)
|
||||
return (mi_msecs_t)clock();
|
||||
#elif (CLOCKS_PER_SEC < 1000)
|
||||
return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC);
|
||||
#else
|
||||
return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000);
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
//----------------------------------------------------------------
|
||||
// Process info
|
||||
//----------------------------------------------------------------
|
||||
|
||||
void _mi_prim_process_info(mi_process_info_t* pinfo)
|
||||
{
|
||||
// use defaults
|
||||
MI_UNUSED(pinfo);
|
||||
}
|
||||
|
||||
|
||||
//----------------------------------------------------------------
|
||||
// Output
|
||||
//----------------------------------------------------------------
|
||||
|
||||
void _mi_prim_out_stderr( const char* msg ) {
|
||||
fputs(msg,stderr);
|
||||
}
|
||||
|
||||
|
||||
//----------------------------------------------------------------
|
||||
// Environment
|
||||
//----------------------------------------------------------------
|
||||
|
||||
bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {
|
||||
// cannot call getenv() when still initializing the C runtime.
|
||||
if (_mi_preloading()) return false;
|
||||
const char* s = getenv(name);
|
||||
if (s == NULL) {
|
||||
// we check the upper case name too.
|
||||
char buf[64+1];
|
||||
size_t len = _mi_strnlen(name,sizeof(buf)-1);
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
buf[i] = _mi_toupper(name[i]);
|
||||
}
|
||||
buf[len] = 0;
|
||||
s = getenv(buf);
|
||||
}
|
||||
if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return false;
|
||||
_mi_strlcpy(result, s, result_size);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
//----------------------------------------------------------------
|
||||
// Random
|
||||
//----------------------------------------------------------------
|
||||
|
||||
bool _mi_prim_random_buf(void* buf, size_t buf_len) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
//----------------------------------------------------------------
|
||||
// Thread init/done
|
||||
//----------------------------------------------------------------
|
||||
|
||||
void _mi_prim_thread_init_auto_done(void) {
|
||||
// nothing
|
||||
}
|
||||
|
||||
void _mi_prim_thread_done_auto_done(void) {
|
||||
// nothing
|
||||
}
|
||||
|
||||
void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
|
||||
MI_UNUSED(theap);
|
||||
}
|
||||
|
||||
bool _mi_prim_thread_is_in_threadpool(void) {
|
||||
return false;
|
||||
}
|
||||
|
||||
void _mi_prim_thread_yield(void) {
|
||||
sleep(0);
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
<WindowsPerformanceRecorder Version="1.0">
|
||||
<Profiles>
|
||||
<SystemCollector Id="WPR_initiated_WprApp_WPR_System_Collector" Name="WPR_initiated_WprApp_WPR System Collector">
|
||||
<BufferSize Value="1024" />
|
||||
<Buffers Value="100" />
|
||||
</SystemCollector>
|
||||
<EventCollector Id="Mimalloc_Collector" Name="Mimalloc Collector">
|
||||
<BufferSize Value="1024" />
|
||||
<Buffers Value="100" />
|
||||
</EventCollector>
|
||||
<SystemProvider Id="WPR_initiated_WprApp_WPR_System_Collector_Provider">
|
||||
<Keywords>
|
||||
<Keyword Value="Loader" />
|
||||
</Keywords>
|
||||
</SystemProvider>
|
||||
<EventProvider Id="MimallocEventProvider" Name="138f4dbb-ee04-4899-aa0a-572ad4475779" NonPagedMemory="true" Stack="true">
|
||||
<EventFilters FilterIn="true">
|
||||
<EventId Value="100" />
|
||||
<EventId Value="101" />
|
||||
</EventFilters>
|
||||
</EventProvider>
|
||||
<Profile Id="CustomHeap.Verbose.File" Name="CustomHeap" Description="RunningProfile:CustomHeap.Verbose.File" LoggingMode="File" DetailLevel="Verbose">
|
||||
<ProblemCategories>
|
||||
<ProblemCategory Value="Resource Analysis" />
|
||||
</ProblemCategories>
|
||||
<Collectors>
|
||||
<SystemCollectorId Value="WPR_initiated_WprApp_WPR_System_Collector">
|
||||
<SystemProviderId Value="WPR_initiated_WprApp_WPR_System_Collector_Provider" />
|
||||
</SystemCollectorId>
|
||||
<EventCollectorId Value="Mimalloc_Collector">
|
||||
<EventProviders>
|
||||
<EventProviderId Value="MimallocEventProvider" >
|
||||
<Keywords>
|
||||
<Keyword Value="100"/>
|
||||
<Keyword Value="101"/>
|
||||
</Keywords>
|
||||
</EventProviderId>
|
||||
</EventProviders>
|
||||
</EventCollectorId>
|
||||
</Collectors>
|
||||
<TraceMergeProperties>
|
||||
<TraceMergeProperty Id="BaseVerboseTraceMergeProperties" Name="BaseTraceMergeProperties">
|
||||
<DeletePreMergedTraceFiles Value="true" />
|
||||
<FileCompression Value="false" />
|
||||
<InjectOnly Value="false" />
|
||||
<SkipMerge Value="false" />
|
||||
<CustomEvents>
|
||||
<CustomEvent Value="ImageId" />
|
||||
<CustomEvent Value="BuildInfo" />
|
||||
<CustomEvent Value="VolumeMapping" />
|
||||
<CustomEvent Value="EventMetadata" />
|
||||
<CustomEvent Value="PerfTrackMetadata" />
|
||||
<CustomEvent Value="WinSAT" />
|
||||
<CustomEvent Value="NetworkInterface" />
|
||||
</CustomEvents>
|
||||
</TraceMergeProperty>
|
||||
</TraceMergeProperties>
|
||||
</Profile>
|
||||
</Profiles>
|
||||
</WindowsPerformanceRecorder>
|
||||
|
||||
@@ -0,0 +1,905 @@
|
||||
//**********************************************************************`
|
||||
//* This is an include file generated by Message Compiler. *`
|
||||
//* *`
|
||||
//* Copyright (c) Microsoft Corporation. All Rights Reserved. *`
|
||||
//**********************************************************************`
|
||||
#pragma once
|
||||
|
||||
//*****************************************************************************
|
||||
//
|
||||
// Notes on the ETW event code generated by MC:
|
||||
//
|
||||
// - Structures and arrays of structures are treated as an opaque binary blob.
|
||||
// The caller is responsible for packing the data for the structure into a
|
||||
// single region of memory, with no padding between values. The macro will
|
||||
// have an extra parameter for the length of the blob.
|
||||
// - Arrays of nul-terminated strings must be packed by the caller into a
|
||||
// single binary blob containing the correct number of strings, with a nul
|
||||
// after each string. The size of the blob is specified in characters, and
|
||||
// includes the final nul.
|
||||
// - Arrays of SID are treated as a single binary blob. The caller is
|
||||
// responsible for packing the SID values into a single region of memory with
|
||||
// no padding.
|
||||
// - The length attribute on the data element in the manifest is significant
|
||||
// for values with intype win:UnicodeString, win:AnsiString, or win:Binary.
|
||||
// The length attribute must be specified for win:Binary, and is optional for
|
||||
// win:UnicodeString and win:AnsiString (if no length is given, the strings
|
||||
// are assumed to be nul-terminated). For win:UnicodeString, the length is
|
||||
// measured in characters, not bytes.
|
||||
// - For an array of win:UnicodeString, win:AnsiString, or win:Binary, the
|
||||
// length attribute applies to every value in the array, so every value in
|
||||
// the array must have the same length. The values in the array are provided
|
||||
// to the macro via a single pointer -- the caller is responsible for packing
|
||||
// all of the values into a single region of memory with no padding between
|
||||
// values.
|
||||
// - Values of type win:CountedUnicodeString, win:CountedAnsiString, and
|
||||
// win:CountedBinary can be generated and collected on Vista or later.
|
||||
// However, they may not decode properly without the Windows 10 2018 Fall
|
||||
// Update.
|
||||
// - Arrays of type win:CountedUnicodeString, win:CountedAnsiString, and
|
||||
// win:CountedBinary must be packed by the caller into a single region of
|
||||
// memory. The format for each item is a UINT16 byte-count followed by that
|
||||
// many bytes of data. When providing the array to the generated macro, you
|
||||
// must provide the total size of the packed array data, including the UINT16
|
||||
// sizes for each item. In the case of win:CountedUnicodeString, the data
|
||||
// size is specified in WCHAR (16-bit) units. In the case of
|
||||
// win:CountedAnsiString and win:CountedBinary, the data size is specified in
|
||||
// bytes.
|
||||
//
|
||||
//*****************************************************************************
|
||||
|
||||
#include <wmistr.h>
|
||||
#include <evntrace.h>
|
||||
#include <evntprov.h>
|
||||
|
||||
#ifndef ETW_INLINE
|
||||
#ifdef _ETW_KM_
|
||||
// In kernel mode, save stack space by never inlining templates.
|
||||
#define ETW_INLINE DECLSPEC_NOINLINE __inline
|
||||
#else
|
||||
// In user mode, save code size by inlining templates as appropriate.
|
||||
#define ETW_INLINE __inline
|
||||
#endif
|
||||
#endif // ETW_INLINE
|
||||
|
||||
#if defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
//
|
||||
// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro:
|
||||
// Define this macro to have the compiler skip the generated functions in this
|
||||
// header.
|
||||
//
|
||||
#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION
|
||||
|
||||
//
|
||||
// MCGEN_USE_KERNEL_MODE_APIS macro:
|
||||
// Controls whether the generated code uses kernel-mode or user-mode APIs.
|
||||
// - Set to 0 to use Windows user-mode APIs such as EventRegister.
|
||||
// - Set to 1 to use Windows kernel-mode APIs such as EtwRegister.
|
||||
// Default is based on whether the _ETW_KM_ macro is defined (i.e. by wdm.h).
|
||||
// Note that the APIs can also be overridden directly, e.g. by setting the
|
||||
// MCGEN_EVENTWRITETRANSFER or MCGEN_EVENTREGISTER macros.
|
||||
//
|
||||
#ifndef MCGEN_USE_KERNEL_MODE_APIS
|
||||
#ifdef _ETW_KM_
|
||||
#define MCGEN_USE_KERNEL_MODE_APIS 1
|
||||
#else
|
||||
#define MCGEN_USE_KERNEL_MODE_APIS 0
|
||||
#endif
|
||||
#endif // MCGEN_USE_KERNEL_MODE_APIS
|
||||
|
||||
//
|
||||
// MCGEN_HAVE_EVENTSETINFORMATION macro:
|
||||
// Controls how McGenEventSetInformation uses the EventSetInformation API.
|
||||
// - Set to 0 to disable the use of EventSetInformation
|
||||
// (McGenEventSetInformation will always return an error).
|
||||
// - Set to 1 to directly invoke MCGEN_EVENTSETINFORMATION.
|
||||
// - Set to 2 to to locate EventSetInformation at runtime via GetProcAddress
|
||||
// (user-mode) or MmGetSystemRoutineAddress (kernel-mode).
|
||||
// Default is determined as follows:
|
||||
// - If MCGEN_EVENTSETINFORMATION has been customized, set to 1
|
||||
// (i.e. use MCGEN_EVENTSETINFORMATION).
|
||||
// - Else if the target OS version has EventSetInformation, set to 1
|
||||
// (i.e. use MCGEN_EVENTSETINFORMATION).
|
||||
// - Else set to 2 (i.e. try to dynamically locate EventSetInformation).
|
||||
// Note that an McGenEventSetInformation function will only be generated if one
|
||||
// or more provider in a manifest has provider traits.
|
||||
//
|
||||
#ifndef MCGEN_HAVE_EVENTSETINFORMATION
|
||||
#ifdef MCGEN_EVENTSETINFORMATION // if MCGEN_EVENTSETINFORMATION has been customized,
|
||||
#define MCGEN_HAVE_EVENTSETINFORMATION 1 // directly invoke MCGEN_EVENTSETINFORMATION(...).
|
||||
#elif MCGEN_USE_KERNEL_MODE_APIS // else if using kernel-mode APIs,
|
||||
#if NTDDI_VERSION >= 0x06040000 // if target OS is Windows 10 or later,
|
||||
#define MCGEN_HAVE_EVENTSETINFORMATION 1 // directly invoke MCGEN_EVENTSETINFORMATION(...).
|
||||
#else // else
|
||||
#define MCGEN_HAVE_EVENTSETINFORMATION 2 // find "EtwSetInformation" via MmGetSystemRoutineAddress.
|
||||
#endif // else (using user-mode APIs)
|
||||
#else // if target OS and SDK is Windows 8 or later,
|
||||
#if WINVER >= 0x0602 && defined(EVENT_FILTER_TYPE_SCHEMATIZED)
|
||||
#define MCGEN_HAVE_EVENTSETINFORMATION 1 // directly invoke MCGEN_EVENTSETINFORMATION(...).
|
||||
#else // else
|
||||
#define MCGEN_HAVE_EVENTSETINFORMATION 2 // find "EventSetInformation" via GetModuleHandleExW/GetProcAddress.
|
||||
#endif
|
||||
#endif
|
||||
#endif // MCGEN_HAVE_EVENTSETINFORMATION
|
||||
|
||||
//
|
||||
// MCGEN Override Macros
|
||||
//
|
||||
// The following override macros may be defined before including this header
|
||||
// to control the APIs used by this header:
|
||||
//
|
||||
// - MCGEN_EVENTREGISTER
|
||||
// - MCGEN_EVENTUNREGISTER
|
||||
// - MCGEN_EVENTSETINFORMATION
|
||||
// - MCGEN_EVENTWRITETRANSFER
|
||||
//
|
||||
// If the the macro is undefined, the MC implementation will default to the
|
||||
// corresponding ETW APIs. For example, if the MCGEN_EVENTREGISTER macro is
|
||||
// undefined, the EventRegister[MyProviderName] macro will use EventRegister
|
||||
// in user mode and will use EtwRegister in kernel mode.
|
||||
//
|
||||
// To prevent issues from conflicting definitions of these macros, the value
|
||||
// of the override macro will be used as a suffix in certain internal function
|
||||
// names. Because of this, the override macros must follow certain rules:
|
||||
//
|
||||
// - The macro must be defined before any MC-generated header is included and
|
||||
// must not be undefined or redefined after any MC-generated header is
|
||||
// included. Different translation units (i.e. different .c or .cpp files)
|
||||
// may set the macros to different values, but within a translation unit
|
||||
// (within a single .c or .cpp file), the macro must be set once and not
|
||||
// changed.
|
||||
// - The override must be an object-like macro, not a function-like macro
|
||||
// (i.e. the override macro must not have a parameter list).
|
||||
// - The override macro's value must be a simple identifier, i.e. must be
|
||||
// something that starts with a letter or '_' and contains only letters,
|
||||
// numbers, and '_' characters.
|
||||
// - If the override macro's value is the name of a second object-like macro,
|
||||
// the second object-like macro must follow the same rules. (The override
|
||||
// macro's value can also be the name of a function-like macro, in which
|
||||
// case the function-like macro does not need to follow the same rules.)
|
||||
//
|
||||
// For example, the following will cause compile errors:
|
||||
//
|
||||
// #define MCGEN_EVENTWRITETRANSFER MyNamespace::MyClass::MyFunction // Value has non-identifier characters (colon).
|
||||
// #define MCGEN_EVENTWRITETRANSFER GetEventWriteFunctionPointer(7) // Value has non-identifier characters (parentheses).
|
||||
// #define MCGEN_EVENTWRITETRANSFER(h,e,a,r,c,d) EventWrite(h,e,c,d) // Override is defined as a function-like macro.
|
||||
// #define MY_OBJECT_LIKE_MACRO MyNamespace::MyClass::MyEventWriteFunction
|
||||
// #define MCGEN_EVENTWRITETRANSFER MY_OBJECT_LIKE_MACRO // Evaluates to something with non-identifier characters (colon).
|
||||
//
|
||||
// The following would be ok:
|
||||
//
|
||||
// #define MCGEN_EVENTWRITETRANSFER MyEventWriteFunction1 // OK, suffix will be "MyEventWriteFunction1".
|
||||
// #define MY_OBJECT_LIKE_MACRO MyEventWriteFunction2
|
||||
// #define MCGEN_EVENTWRITETRANSFER MY_OBJECT_LIKE_MACRO // OK, suffix will be "MyEventWriteFunction2".
|
||||
// #define MY_FUNCTION_LIKE_MACRO(h,e,a,r,c,d) MyNamespace::MyClass::MyEventWriteFunction3(h,e,c,d)
|
||||
// #define MCGEN_EVENTWRITETRANSFER MY_FUNCTION_LIKE_MACRO // OK, suffix will be "MY_FUNCTION_LIKE_MACRO".
|
||||
//
|
||||
#ifndef MCGEN_EVENTREGISTER
|
||||
#if MCGEN_USE_KERNEL_MODE_APIS
|
||||
#define MCGEN_EVENTREGISTER EtwRegister
|
||||
#else
|
||||
#define MCGEN_EVENTREGISTER EventRegister
|
||||
#endif
|
||||
#endif // MCGEN_EVENTREGISTER
|
||||
#ifndef MCGEN_EVENTUNREGISTER
|
||||
#if MCGEN_USE_KERNEL_MODE_APIS
|
||||
#define MCGEN_EVENTUNREGISTER EtwUnregister
|
||||
#else
|
||||
#define MCGEN_EVENTUNREGISTER EventUnregister
|
||||
#endif
|
||||
#endif // MCGEN_EVENTUNREGISTER
|
||||
#ifndef MCGEN_EVENTSETINFORMATION
|
||||
#if MCGEN_USE_KERNEL_MODE_APIS
|
||||
#define MCGEN_EVENTSETINFORMATION EtwSetInformation
|
||||
#else
|
||||
#define MCGEN_EVENTSETINFORMATION EventSetInformation
|
||||
#endif
|
||||
#endif // MCGEN_EVENTSETINFORMATION
|
||||
#ifndef MCGEN_EVENTWRITETRANSFER
|
||||
#if MCGEN_USE_KERNEL_MODE_APIS
|
||||
#define MCGEN_EVENTWRITETRANSFER EtwWriteTransfer
|
||||
#else
|
||||
#define MCGEN_EVENTWRITETRANSFER EventWriteTransfer
|
||||
#endif
|
||||
#endif // MCGEN_EVENTWRITETRANSFER
|
||||
|
||||
//
|
||||
// MCGEN_EVENT_ENABLED macro:
|
||||
// Override to control how the EventWrite[EventName] macros determine whether
|
||||
// an event is enabled. The default behavior is for EventWrite[EventName] to
|
||||
// use the EventEnabled[EventName] macros.
|
||||
//
|
||||
#ifndef MCGEN_EVENT_ENABLED
|
||||
#define MCGEN_EVENT_ENABLED(EventName) EventEnabled##EventName()
|
||||
#endif
|
||||
|
||||
//
|
||||
// MCGEN_EVENT_ENABLED_FORCONTEXT macro:
|
||||
// Override to control how the EventWrite[EventName]_ForContext macros
|
||||
// determine whether an event is enabled. The default behavior is for
|
||||
// EventWrite[EventName]_ForContext to use the
|
||||
// EventEnabled[EventName]_ForContext macros.
|
||||
//
|
||||
#ifndef MCGEN_EVENT_ENABLED_FORCONTEXT
|
||||
#define MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, EventName) EventEnabled##EventName##_ForContext(pContext)
|
||||
#endif
|
||||
|
||||
//
|
||||
// MCGEN_ENABLE_CHECK macro:
|
||||
// Determines whether the specified event would be considered as enabled
|
||||
// based on the state of the specified context. Slightly faster than calling
|
||||
// McGenEventEnabled directly.
|
||||
//
|
||||
#ifndef MCGEN_ENABLE_CHECK
|
||||
#define MCGEN_ENABLE_CHECK(Context, Descriptor) (Context.IsEnabled && McGenEventEnabled(&Context, &Descriptor))
|
||||
#endif
|
||||
|
||||
#if !defined(MCGEN_TRACE_CONTEXT_DEF)
|
||||
#define MCGEN_TRACE_CONTEXT_DEF
|
||||
// This structure is for use by MC-generated code and should not be used directly.
|
||||
typedef struct _MCGEN_TRACE_CONTEXT
|
||||
{
|
||||
TRACEHANDLE RegistrationHandle;
|
||||
TRACEHANDLE Logger; // Used as pointer to provider traits.
|
||||
ULONGLONG MatchAnyKeyword;
|
||||
ULONGLONG MatchAllKeyword;
|
||||
ULONG Flags;
|
||||
ULONG IsEnabled;
|
||||
UCHAR Level;
|
||||
UCHAR Reserve;
|
||||
USHORT EnableBitsCount;
|
||||
PULONG EnableBitMask;
|
||||
const ULONGLONG* EnableKeyWords;
|
||||
const UCHAR* EnableLevel;
|
||||
} MCGEN_TRACE_CONTEXT, *PMCGEN_TRACE_CONTEXT;
|
||||
#endif // MCGEN_TRACE_CONTEXT_DEF
|
||||
|
||||
#if !defined(MCGEN_LEVEL_KEYWORD_ENABLED_DEF)
|
||||
#define MCGEN_LEVEL_KEYWORD_ENABLED_DEF
|
||||
//
|
||||
// Determines whether an event with a given Level and Keyword would be
|
||||
// considered as enabled based on the state of the specified context.
|
||||
// Note that you may want to use MCGEN_ENABLE_CHECK instead of calling this
|
||||
// function directly.
|
||||
//
|
||||
FORCEINLINE
|
||||
BOOLEAN
|
||||
McGenLevelKeywordEnabled(
|
||||
_In_ PMCGEN_TRACE_CONTEXT EnableInfo,
|
||||
_In_ UCHAR Level,
|
||||
_In_ ULONGLONG Keyword
|
||||
)
|
||||
{
|
||||
//
|
||||
// Check if the event Level is lower than the level at which
|
||||
// the channel is enabled.
|
||||
// If the event Level is 0 or the channel is enabled at level 0,
|
||||
// all levels are enabled.
|
||||
//
|
||||
|
||||
if ((Level <= EnableInfo->Level) || // This also covers the case of Level == 0.
|
||||
(EnableInfo->Level == 0)) {
|
||||
|
||||
//
|
||||
// Check if Keyword is enabled
|
||||
//
|
||||
|
||||
if ((Keyword == (ULONGLONG)0) ||
|
||||
((Keyword & EnableInfo->MatchAnyKeyword) &&
|
||||
((Keyword & EnableInfo->MatchAllKeyword) == EnableInfo->MatchAllKeyword))) {
|
||||
return TRUE;
|
||||
}
|
||||
}
|
||||
|
||||
return FALSE;
|
||||
}
|
||||
#endif // MCGEN_LEVEL_KEYWORD_ENABLED_DEF
|
||||
|
||||
#if !defined(MCGEN_EVENT_ENABLED_DEF)
|
||||
#define MCGEN_EVENT_ENABLED_DEF
|
||||
//
|
||||
// Determines whether the specified event would be considered as enabled based
|
||||
// on the state of the specified context. Note that you may want to use
|
||||
// MCGEN_ENABLE_CHECK instead of calling this function directly.
|
||||
//
|
||||
FORCEINLINE
|
||||
BOOLEAN
|
||||
McGenEventEnabled(
|
||||
_In_ PMCGEN_TRACE_CONTEXT EnableInfo,
|
||||
_In_ PCEVENT_DESCRIPTOR EventDescriptor
|
||||
)
|
||||
{
|
||||
return McGenLevelKeywordEnabled(EnableInfo, EventDescriptor->Level, EventDescriptor->Keyword);
|
||||
}
|
||||
#endif // MCGEN_EVENT_ENABLED_DEF
|
||||
|
||||
#if !defined(MCGEN_CONTROL_CALLBACK)
|
||||
#define MCGEN_CONTROL_CALLBACK
|
||||
|
||||
// This function is for use by MC-generated code and should not be used directly.
|
||||
DECLSPEC_NOINLINE __inline
|
||||
VOID
|
||||
__stdcall
|
||||
McGenControlCallbackV2(
|
||||
_In_ LPCGUID SourceId,
|
||||
_In_ ULONG ControlCode,
|
||||
_In_ UCHAR Level,
|
||||
_In_ ULONGLONG MatchAnyKeyword,
|
||||
_In_ ULONGLONG MatchAllKeyword,
|
||||
_In_opt_ PEVENT_FILTER_DESCRIPTOR FilterData,
|
||||
_Inout_opt_ PVOID CallbackContext
|
||||
)
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This is the notification callback for Windows Vista and later.
|
||||
|
||||
Arguments:
|
||||
|
||||
SourceId - The GUID that identifies the session that enabled the provider.
|
||||
|
||||
ControlCode - The parameter indicates whether the provider
|
||||
is being enabled or disabled.
|
||||
|
||||
Level - The level at which the event is enabled.
|
||||
|
||||
MatchAnyKeyword - The bitmask of keywords that the provider uses to
|
||||
determine the category of events that it writes.
|
||||
|
||||
MatchAllKeyword - This bitmask additionally restricts the category
|
||||
of events that the provider writes.
|
||||
|
||||
FilterData - The provider-defined data.
|
||||
|
||||
CallbackContext - The context of the callback that is defined when the provider
|
||||
called EtwRegister to register itself.
|
||||
|
||||
Remarks:
|
||||
|
||||
ETW calls this function to notify provider of enable/disable
|
||||
|
||||
--*/
|
||||
{
|
||||
PMCGEN_TRACE_CONTEXT Ctx = (PMCGEN_TRACE_CONTEXT)CallbackContext;
|
||||
ULONG Ix;
|
||||
#ifndef MCGEN_PRIVATE_ENABLE_CALLBACK_V2
|
||||
UNREFERENCED_PARAMETER(SourceId);
|
||||
UNREFERENCED_PARAMETER(FilterData);
|
||||
#endif
|
||||
|
||||
if (Ctx == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
switch (ControlCode) {
|
||||
|
||||
case EVENT_CONTROL_CODE_ENABLE_PROVIDER:
|
||||
Ctx->Level = Level;
|
||||
Ctx->MatchAnyKeyword = MatchAnyKeyword;
|
||||
Ctx->MatchAllKeyword = MatchAllKeyword;
|
||||
Ctx->IsEnabled = EVENT_CONTROL_CODE_ENABLE_PROVIDER;
|
||||
|
||||
for (Ix = 0; Ix < Ctx->EnableBitsCount; Ix += 1) {
|
||||
if (McGenLevelKeywordEnabled(Ctx, Ctx->EnableLevel[Ix], Ctx->EnableKeyWords[Ix]) != FALSE) {
|
||||
Ctx->EnableBitMask[Ix >> 5] |= (1 << (Ix % 32));
|
||||
} else {
|
||||
Ctx->EnableBitMask[Ix >> 5] &= ~(1 << (Ix % 32));
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case EVENT_CONTROL_CODE_DISABLE_PROVIDER:
|
||||
Ctx->IsEnabled = EVENT_CONTROL_CODE_DISABLE_PROVIDER;
|
||||
Ctx->Level = 0;
|
||||
Ctx->MatchAnyKeyword = 0;
|
||||
Ctx->MatchAllKeyword = 0;
|
||||
if (Ctx->EnableBitsCount > 0) {
|
||||
#pragma warning(suppress: 26451) // Arithmetic overflow cannot occur, no matter the value of EnableBitCount
|
||||
RtlZeroMemory(Ctx->EnableBitMask, (((Ctx->EnableBitsCount - 1) / 32) + 1) * sizeof(ULONG));
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
#ifdef MCGEN_PRIVATE_ENABLE_CALLBACK_V2
|
||||
//
|
||||
// Call user defined callback
|
||||
//
|
||||
MCGEN_PRIVATE_ENABLE_CALLBACK_V2(
|
||||
SourceId,
|
||||
ControlCode,
|
||||
Level,
|
||||
MatchAnyKeyword,
|
||||
MatchAllKeyword,
|
||||
FilterData,
|
||||
CallbackContext
|
||||
);
|
||||
#endif // MCGEN_PRIVATE_ENABLE_CALLBACK_V2
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
#endif // MCGEN_CONTROL_CALLBACK
|
||||
|
||||
#ifndef _mcgen_PENABLECALLBACK
|
||||
#if MCGEN_USE_KERNEL_MODE_APIS
|
||||
#define _mcgen_PENABLECALLBACK PETWENABLECALLBACK
|
||||
#else
|
||||
#define _mcgen_PENABLECALLBACK PENABLECALLBACK
|
||||
#endif
|
||||
#endif // _mcgen_PENABLECALLBACK
|
||||
|
||||
#if !defined(_mcgen_PASTE2)
|
||||
// This macro is for use by MC-generated code and should not be used directly.
|
||||
#define _mcgen_PASTE2(a, b) _mcgen_PASTE2_imp(a, b)
|
||||
#define _mcgen_PASTE2_imp(a, b) a##b
|
||||
#endif // _mcgen_PASTE2
|
||||
|
||||
#if !defined(_mcgen_PASTE3)
|
||||
// This macro is for use by MC-generated code and should not be used directly.
|
||||
#define _mcgen_PASTE3(a, b, c) _mcgen_PASTE3_imp(a, b, c)
|
||||
#define _mcgen_PASTE3_imp(a, b, c) a##b##_##c
|
||||
#endif // _mcgen_PASTE3
|
||||
|
||||
//
|
||||
// Macro validation
|
||||
//
|
||||
|
||||
// Validate MCGEN_EVENTREGISTER:
|
||||
|
||||
// Trigger an error if MCGEN_EVENTREGISTER is not an unqualified (simple) identifier:
|
||||
struct _mcgen_PASTE2(MCGEN_EVENTREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTREGISTER);
|
||||
|
||||
// Trigger an error if MCGEN_EVENTREGISTER is redefined:
|
||||
typedef struct _mcgen_PASTE2(MCGEN_EVENTREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTREGISTER)
|
||||
MCGEN_EVENTREGISTER_must_not_be_redefined_between_headers;
|
||||
|
||||
// Trigger an error if MCGEN_EVENTREGISTER is defined as a function-like macro:
|
||||
typedef void MCGEN_EVENTREGISTER_must_not_be_a_functionLike_macro_MCGEN_EVENTREGISTER;
|
||||
typedef int _mcgen_PASTE2(MCGEN_EVENTREGISTER_must_not_be_a_functionLike_macro_, MCGEN_EVENTREGISTER);
|
||||
|
||||
// Validate MCGEN_EVENTUNREGISTER:
|
||||
|
||||
// Trigger an error if MCGEN_EVENTUNREGISTER is not an unqualified (simple) identifier:
|
||||
struct _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTUNREGISTER);
|
||||
|
||||
// Trigger an error if MCGEN_EVENTUNREGISTER is redefined:
|
||||
typedef struct _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTUNREGISTER)
|
||||
MCGEN_EVENTUNREGISTER_must_not_be_redefined_between_headers;
|
||||
|
||||
// Trigger an error if MCGEN_EVENTUNREGISTER is defined as a function-like macro:
|
||||
typedef void MCGEN_EVENTUNREGISTER_must_not_be_a_functionLike_macro_MCGEN_EVENTUNREGISTER;
|
||||
typedef int _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_must_not_be_a_functionLike_macro_, MCGEN_EVENTUNREGISTER);
|
||||
|
||||
// Validate MCGEN_EVENTSETINFORMATION:
|
||||
|
||||
// Trigger an error if MCGEN_EVENTSETINFORMATION is not an unqualified (simple) identifier:
|
||||
struct _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTSETINFORMATION);
|
||||
|
||||
// Trigger an error if MCGEN_EVENTSETINFORMATION is redefined:
|
||||
typedef struct _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTSETINFORMATION)
|
||||
MCGEN_EVENTSETINFORMATION_must_not_be_redefined_between_headers;
|
||||
|
||||
// Trigger an error if MCGEN_EVENTSETINFORMATION is defined as a function-like macro:
|
||||
typedef void MCGEN_EVENTSETINFORMATION_must_not_be_a_functionLike_macro_MCGEN_EVENTSETINFORMATION;
|
||||
typedef int _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_must_not_be_a_functionLike_macro_, MCGEN_EVENTSETINFORMATION);
|
||||
|
||||
// Validate MCGEN_EVENTWRITETRANSFER:
|
||||
|
||||
// Trigger an error if MCGEN_EVENTWRITETRANSFER is not an unqualified (simple) identifier:
|
||||
struct _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTWRITETRANSFER);
|
||||
|
||||
// Trigger an error if MCGEN_EVENTWRITETRANSFER is redefined:
|
||||
typedef struct _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTWRITETRANSFER)
|
||||
MCGEN_EVENTWRITETRANSFER_must_not_be_redefined_between_headers;;
|
||||
|
||||
// Trigger an error if MCGEN_EVENTWRITETRANSFER is defined as a function-like macro:
|
||||
typedef void MCGEN_EVENTWRITETRANSFER_must_not_be_a_functionLike_macro_MCGEN_EVENTWRITETRANSFER;
|
||||
typedef int _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_must_not_be_a_functionLike_macro_, MCGEN_EVENTWRITETRANSFER);
|
||||
|
||||
#ifndef McGenEventWrite_def
|
||||
#define McGenEventWrite_def
|
||||
|
||||
// This macro is for use by MC-generated code and should not be used directly.
|
||||
#define McGenEventWrite _mcgen_PASTE2(McGenEventWrite_, MCGEN_EVENTWRITETRANSFER)
|
||||
|
||||
// This function is for use by MC-generated code and should not be used directly.
|
||||
DECLSPEC_NOINLINE __inline
|
||||
ULONG __stdcall
|
||||
McGenEventWrite(
|
||||
_In_ PMCGEN_TRACE_CONTEXT Context,
|
||||
_In_ PCEVENT_DESCRIPTOR Descriptor,
|
||||
_In_opt_ LPCGUID ActivityId,
|
||||
_In_range_(1, 128) ULONG EventDataCount,
|
||||
_Pre_cap_(EventDataCount) EVENT_DATA_DESCRIPTOR* EventData
|
||||
)
|
||||
{
|
||||
const USHORT UNALIGNED* Traits;
|
||||
|
||||
// Some customized MCGEN_EVENTWRITETRANSFER macros might ignore ActivityId.
|
||||
UNREFERENCED_PARAMETER(ActivityId);
|
||||
|
||||
Traits = (const USHORT UNALIGNED*)(UINT_PTR)Context->Logger;
|
||||
|
||||
if (Traits == NULL) {
|
||||
EventData[0].Ptr = 0;
|
||||
EventData[0].Size = 0;
|
||||
EventData[0].Reserved = 0;
|
||||
} else {
|
||||
EventData[0].Ptr = (ULONG_PTR)Traits;
|
||||
EventData[0].Size = *Traits;
|
||||
EventData[0].Reserved = 2; // EVENT_DATA_DESCRIPTOR_TYPE_PROVIDER_METADATA
|
||||
}
|
||||
|
||||
return MCGEN_EVENTWRITETRANSFER(
|
||||
Context->RegistrationHandle,
|
||||
Descriptor,
|
||||
ActivityId,
|
||||
NULL,
|
||||
EventDataCount,
|
||||
EventData);
|
||||
}
|
||||
#endif // McGenEventWrite_def
|
||||
|
||||
#if !defined(McGenEventRegisterUnregister)
|
||||
#define McGenEventRegisterUnregister
|
||||
|
||||
// This macro is for use by MC-generated code and should not be used directly.
|
||||
#define McGenEventRegister _mcgen_PASTE2(McGenEventRegister_, MCGEN_EVENTREGISTER)
|
||||
|
||||
#pragma warning(push)
|
||||
#pragma warning(disable:6103)
|
||||
// This function is for use by MC-generated code and should not be used directly.
|
||||
DECLSPEC_NOINLINE __inline
|
||||
ULONG __stdcall
|
||||
McGenEventRegister(
|
||||
_In_ LPCGUID ProviderId,
|
||||
_In_opt_ _mcgen_PENABLECALLBACK EnableCallback,
|
||||
_In_opt_ PVOID CallbackContext,
|
||||
_Inout_ PREGHANDLE RegHandle
|
||||
)
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This function registers the provider with ETW.
|
||||
|
||||
Arguments:
|
||||
|
||||
ProviderId - Provider ID to register with ETW.
|
||||
|
||||
EnableCallback - Callback to be used.
|
||||
|
||||
CallbackContext - Context for the callback.
|
||||
|
||||
RegHandle - Pointer to registration handle.
|
||||
|
||||
Remarks:
|
||||
|
||||
Should not be called if the provider is already registered (i.e. should not
|
||||
be called if *RegHandle != 0). Repeatedly registering a provider is a bug
|
||||
and may indicate a race condition. However, for compatibility with previous
|
||||
behavior, this function will return SUCCESS in this case.
|
||||
|
||||
--*/
|
||||
{
|
||||
ULONG Error;
|
||||
|
||||
if (*RegHandle != 0)
|
||||
{
|
||||
Error = 0; // ERROR_SUCCESS
|
||||
}
|
||||
else
|
||||
{
|
||||
Error = MCGEN_EVENTREGISTER(ProviderId, EnableCallback, CallbackContext, RegHandle);
|
||||
}
|
||||
|
||||
return Error;
|
||||
}
|
||||
#pragma warning(pop)
|
||||
|
||||
// This macro is for use by MC-generated code and should not be used directly.
|
||||
#define McGenEventUnregister _mcgen_PASTE2(McGenEventUnregister_, MCGEN_EVENTUNREGISTER)
|
||||
|
||||
// This function is for use by MC-generated code and should not be used directly.
|
||||
DECLSPEC_NOINLINE __inline
|
||||
ULONG __stdcall
|
||||
McGenEventUnregister(_Inout_ PREGHANDLE RegHandle)
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
Unregister from ETW and set *RegHandle = 0.
|
||||
|
||||
Arguments:
|
||||
|
||||
RegHandle - the pointer to the provider registration handle
|
||||
|
||||
Remarks:
|
||||
|
||||
If provider has not been registered (i.e. if *RegHandle == 0),
|
||||
return SUCCESS. It is safe to call McGenEventUnregister even if the
|
||||
call to McGenEventRegister returned an error.
|
||||
|
||||
--*/
|
||||
{
|
||||
ULONG Error;
|
||||
|
||||
if(*RegHandle == 0)
|
||||
{
|
||||
Error = 0; // ERROR_SUCCESS
|
||||
}
|
||||
else
|
||||
{
|
||||
Error = MCGEN_EVENTUNREGISTER(*RegHandle);
|
||||
*RegHandle = (REGHANDLE)0;
|
||||
}
|
||||
|
||||
return Error;
|
||||
}
|
||||
|
||||
#endif // McGenEventRegisterUnregister
|
||||
|
||||
#ifndef _mcgen_EVENT_BIT_SET
|
||||
#if defined(_M_IX86) || defined(_M_X64)
|
||||
// This macro is for use by MC-generated code and should not be used directly.
|
||||
#define _mcgen_EVENT_BIT_SET(EnableBits, BitPosition) ((((const unsigned char*)EnableBits)[BitPosition >> 3] & (1u << (BitPosition & 7))) != 0)
|
||||
#else // CPU type
|
||||
// This macro is for use by MC-generated code and should not be used directly.
|
||||
#define _mcgen_EVENT_BIT_SET(EnableBits, BitPosition) ((EnableBits[BitPosition >> 5] & (1u << (BitPosition & 31))) != 0)
|
||||
#endif // CPU type
|
||||
#endif // _mcgen_EVENT_BIT_SET
|
||||
|
||||
#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION
|
||||
|
||||
//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
// Provider "microsoft-windows-mimalloc" event count 2
|
||||
//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
|
||||
// Provider GUID = 138f4dbb-ee04-4899-aa0a-572ad4475779
|
||||
EXTERN_C __declspec(selectany) const GUID ETW_MI_Provider = {0x138f4dbb, 0xee04, 0x4899, {0xaa, 0x0a, 0x57, 0x2a, 0xd4, 0x47, 0x57, 0x79}};
|
||||
|
||||
#ifndef ETW_MI_Provider_Traits
|
||||
#define ETW_MI_Provider_Traits NULL
|
||||
#endif // ETW_MI_Provider_Traits
|
||||
|
||||
//
|
||||
// Event Descriptors
|
||||
//
|
||||
EXTERN_C __declspec(selectany) const EVENT_DESCRIPTOR ETW_MI_ALLOC = {0x64, 0x1, 0x0, 0x4, 0x0, 0x0, 0x0};
|
||||
#define ETW_MI_ALLOC_value 0x64
|
||||
EXTERN_C __declspec(selectany) const EVENT_DESCRIPTOR ETW_MI_FREE = {0x65, 0x1, 0x0, 0x4, 0x0, 0x0, 0x0};
|
||||
#define ETW_MI_FREE_value 0x65
|
||||
|
||||
//
|
||||
// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro:
|
||||
// Define this macro to have the compiler skip the generated functions in this
|
||||
// header.
|
||||
//
|
||||
#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION
|
||||
|
||||
//
|
||||
// Event Enablement Bits
|
||||
// These variables are for use by MC-generated code and should not be used directly.
|
||||
//
|
||||
EXTERN_C __declspec(selectany) DECLSPEC_CACHEALIGN ULONG microsoft_windows_mimallocEnableBits[1];
|
||||
EXTERN_C __declspec(selectany) const ULONGLONG microsoft_windows_mimallocKeywords[1] = {0x0};
|
||||
EXTERN_C __declspec(selectany) const unsigned char microsoft_windows_mimallocLevels[1] = {4};
|
||||
|
||||
//
|
||||
// Provider context
|
||||
//
|
||||
EXTERN_C __declspec(selectany) MCGEN_TRACE_CONTEXT ETW_MI_Provider_Context = {0, (ULONG_PTR)ETW_MI_Provider_Traits, 0, 0, 0, 0, 0, 0, 1, microsoft_windows_mimallocEnableBits, microsoft_windows_mimallocKeywords, microsoft_windows_mimallocLevels};
|
||||
|
||||
//
|
||||
// Provider REGHANDLE
|
||||
//
|
||||
#define microsoft_windows_mimallocHandle (ETW_MI_Provider_Context.RegistrationHandle)
|
||||
|
||||
//
|
||||
// This macro is set to 0, indicating that the EventWrite[Name] macros do not
|
||||
// have an Activity parameter. This is controlled by the -km and -um options.
|
||||
//
|
||||
#define ETW_MI_Provider_EventWriteActivity 0
|
||||
|
||||
//
|
||||
// Register with ETW using the control GUID specified in the manifest.
|
||||
// Invoke this macro during module initialization (i.e. program startup,
|
||||
// DLL process attach, or driver load) to initialize the provider.
|
||||
// Note that if this function returns an error, the error means that
|
||||
// will not work, but no action needs to be taken -- even if EventRegister
|
||||
// returns an error, it is generally safe to use EventWrite and
|
||||
// EventUnregister macros (they will be no-ops if EventRegister failed).
|
||||
//
|
||||
#ifndef EventRegistermicrosoft_windows_mimalloc
|
||||
#define EventRegistermicrosoft_windows_mimalloc() McGenEventRegister(&ETW_MI_Provider, McGenControlCallbackV2, &ETW_MI_Provider_Context, µsoft_windows_mimallocHandle)
|
||||
#endif
|
||||
|
||||
//
|
||||
// Register with ETW using a specific control GUID (i.e. a GUID other than what
|
||||
// is specified in the manifest). Advanced scenarios only.
|
||||
//
|
||||
#ifndef EventRegisterByGuidmicrosoft_windows_mimalloc
|
||||
#define EventRegisterByGuidmicrosoft_windows_mimalloc(Guid) McGenEventRegister(&(Guid), McGenControlCallbackV2, &ETW_MI_Provider_Context, µsoft_windows_mimallocHandle)
|
||||
#endif
|
||||
|
||||
//
|
||||
// Unregister with ETW and close the provider.
|
||||
// Invoke this macro during module shutdown (i.e. program exit, DLL process
|
||||
// detach, or driver unload) to unregister the provider.
|
||||
// Note that you MUST call EventUnregister before DLL or driver unload
|
||||
// (not optional): failure to unregister a provider before DLL or driver unload
|
||||
// will result in crashes.
|
||||
//
|
||||
#ifndef EventUnregistermicrosoft_windows_mimalloc
|
||||
#define EventUnregistermicrosoft_windows_mimalloc() McGenEventUnregister(µsoft_windows_mimallocHandle)
|
||||
#endif
|
||||
|
||||
//
|
||||
// MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION macro:
|
||||
// Define this macro to enable support for caller-allocated provider context.
|
||||
//
|
||||
#ifdef MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION
|
||||
|
||||
//
|
||||
// Advanced scenarios: Caller-allocated provider context.
|
||||
// Use when multiple differently-configured provider handles are needed,
|
||||
// e.g. for container-aware drivers, one context per container.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// - Caller enables the feature before including this header, e.g.
|
||||
// #define MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION 1
|
||||
// - Caller allocates memory, e.g. pContext = malloc(sizeof(McGenContext_microsoft_windows_mimalloc));
|
||||
// - Caller registers the provider, e.g. EventRegistermicrosoft_windows_mimalloc_ForContext(pContext);
|
||||
// - Caller writes events, e.g. EventWriteMyEvent_ForContext(pContext, ...);
|
||||
// - Caller unregisters, e.g. EventUnregistermicrosoft_windows_mimalloc_ForContext(pContext);
|
||||
// - Caller frees memory, e.g. free(pContext);
|
||||
//
|
||||
|
||||
typedef struct tagMcGenContext_microsoft_windows_mimalloc {
|
||||
// The fields of this structure are subject to change and should
|
||||
// not be accessed directly. To access the provider's REGHANDLE,
|
||||
// use microsoft_windows_mimallocHandle_ForContext(pContext).
|
||||
MCGEN_TRACE_CONTEXT Context;
|
||||
ULONG EnableBits[1];
|
||||
} McGenContext_microsoft_windows_mimalloc;
|
||||
|
||||
#define EventRegistermicrosoft_windows_mimalloc_ForContext(pContext) _mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)(&ETW_MI_Provider, pContext)
|
||||
#define EventRegisterByGuidmicrosoft_windows_mimalloc_ForContext(Guid, pContext) _mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)(&(Guid), pContext)
|
||||
#define EventUnregistermicrosoft_windows_mimalloc_ForContext(pContext) McGenEventUnregister(&(pContext)->Context.RegistrationHandle)
|
||||
|
||||
//
|
||||
// Provider REGHANDLE for caller-allocated context.
|
||||
//
|
||||
#define microsoft_windows_mimallocHandle_ForContext(pContext) ((pContext)->Context.RegistrationHandle)
|
||||
|
||||
// This function is for use by MC-generated code and should not be used directly.
|
||||
// Initialize and register the caller-allocated context.
|
||||
__inline
|
||||
ULONG __stdcall
|
||||
_mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)(
|
||||
_In_ LPCGUID pProviderId,
|
||||
_Out_ McGenContext_microsoft_windows_mimalloc* pContext)
|
||||
{
|
||||
RtlZeroMemory(pContext, sizeof(*pContext));
|
||||
pContext->Context.Logger = (ULONG_PTR)ETW_MI_Provider_Traits;
|
||||
pContext->Context.EnableBitsCount = 1;
|
||||
pContext->Context.EnableBitMask = pContext->EnableBits;
|
||||
pContext->Context.EnableKeyWords = microsoft_windows_mimallocKeywords;
|
||||
pContext->Context.EnableLevel = microsoft_windows_mimallocLevels;
|
||||
return McGenEventRegister(
|
||||
pProviderId,
|
||||
McGenControlCallbackV2,
|
||||
&pContext->Context,
|
||||
&pContext->Context.RegistrationHandle);
|
||||
}
|
||||
|
||||
// This function is for use by MC-generated code and should not be used directly.
|
||||
// Trigger a compile error if called with the wrong parameter type.
|
||||
FORCEINLINE
|
||||
_Ret_ McGenContext_microsoft_windows_mimalloc*
|
||||
_mcgen_CheckContextType_microsoft_windows_mimalloc(_In_ McGenContext_microsoft_windows_mimalloc* pContext)
|
||||
{
|
||||
return pContext;
|
||||
}
|
||||
|
||||
#endif // MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION
|
||||
|
||||
//
|
||||
// Enablement check macro for event "ETW_MI_ALLOC"
|
||||
//
|
||||
#define EventEnabledETW_MI_ALLOC() _mcgen_EVENT_BIT_SET(microsoft_windows_mimallocEnableBits, 0)
|
||||
#define EventEnabledETW_MI_ALLOC_ForContext(pContext) _mcgen_EVENT_BIT_SET(_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->EnableBits, 0)
|
||||
|
||||
//
|
||||
// Event write macros for event "ETW_MI_ALLOC"
|
||||
//
|
||||
#define EventWriteETW_MI_ALLOC(Address, Size) \
|
||||
MCGEN_EVENT_ENABLED(ETW_MI_ALLOC) \
|
||||
? _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&ETW_MI_Provider_Context, &ETW_MI_ALLOC, Address, Size) : 0
|
||||
#define EventWriteETW_MI_ALLOC_AssumeEnabled(Address, Size) \
|
||||
_mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&ETW_MI_Provider_Context, &ETW_MI_ALLOC, Address, Size)
|
||||
#define EventWriteETW_MI_ALLOC_ForContext(pContext, Address, Size) \
|
||||
MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, ETW_MI_ALLOC) \
|
||||
? _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&(pContext)->Context, &ETW_MI_ALLOC, Address, Size) : 0
|
||||
#define EventWriteETW_MI_ALLOC_ForContextAssumeEnabled(pContext, Address, Size) \
|
||||
_mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->Context, &ETW_MI_ALLOC, Address, Size)
|
||||
|
||||
// This macro is for use by MC-generated code and should not be used directly.
|
||||
#define _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC _mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER)
|
||||
|
||||
//
|
||||
// Enablement check macro for event "ETW_MI_FREE"
|
||||
//
|
||||
#define EventEnabledETW_MI_FREE() _mcgen_EVENT_BIT_SET(microsoft_windows_mimallocEnableBits, 0)
|
||||
#define EventEnabledETW_MI_FREE_ForContext(pContext) _mcgen_EVENT_BIT_SET(_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->EnableBits, 0)
|
||||
|
||||
//
|
||||
// Event write macros for event "ETW_MI_FREE"
|
||||
//
|
||||
#define EventWriteETW_MI_FREE(Address, Size) \
|
||||
MCGEN_EVENT_ENABLED(ETW_MI_FREE) \
|
||||
? _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&ETW_MI_Provider_Context, &ETW_MI_FREE, Address, Size) : 0
|
||||
#define EventWriteETW_MI_FREE_AssumeEnabled(Address, Size) \
|
||||
_mcgen_TEMPLATE_FOR_ETW_MI_FREE(&ETW_MI_Provider_Context, &ETW_MI_FREE, Address, Size)
|
||||
#define EventWriteETW_MI_FREE_ForContext(pContext, Address, Size) \
|
||||
MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, ETW_MI_FREE) \
|
||||
? _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&(pContext)->Context, &ETW_MI_FREE, Address, Size) : 0
|
||||
#define EventWriteETW_MI_FREE_ForContextAssumeEnabled(pContext, Address, Size) \
|
||||
_mcgen_TEMPLATE_FOR_ETW_MI_FREE(&_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->Context, &ETW_MI_FREE, Address, Size)
|
||||
|
||||
// This macro is for use by MC-generated code and should not be used directly.
|
||||
#define _mcgen_TEMPLATE_FOR_ETW_MI_FREE _mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER)
|
||||
|
||||
#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION
|
||||
|
||||
//
|
||||
// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro:
|
||||
// Define this macro to have the compiler skip the generated functions in this
|
||||
// header.
|
||||
//
|
||||
#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION
|
||||
|
||||
//
|
||||
// Template Functions
|
||||
//
|
||||
|
||||
//
|
||||
// Function for template "ETW_CUSTOM_HEAP_ALLOC_DATA" (and possibly others).
|
||||
// This function is for use by MC-generated code and should not be used directly.
|
||||
//
|
||||
#ifndef McTemplateU0xx_def
|
||||
#define McTemplateU0xx_def
|
||||
ETW_INLINE
|
||||
ULONG
|
||||
_mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER)(
|
||||
_In_ PMCGEN_TRACE_CONTEXT Context,
|
||||
_In_ PCEVENT_DESCRIPTOR Descriptor,
|
||||
_In_ const unsigned __int64 _Arg0,
|
||||
_In_ const unsigned __int64 _Arg1
|
||||
)
|
||||
{
|
||||
#define McTemplateU0xx_ARGCOUNT 2
|
||||
|
||||
EVENT_DATA_DESCRIPTOR EventData[McTemplateU0xx_ARGCOUNT + 1];
|
||||
|
||||
EventDataDescCreate(&EventData[1],&_Arg0, sizeof(const unsigned __int64) );
|
||||
|
||||
EventDataDescCreate(&EventData[2],&_Arg1, sizeof(const unsigned __int64) );
|
||||
|
||||
return McGenEventWrite(Context, Descriptor, NULL, McTemplateU0xx_ARGCOUNT + 1, EventData);
|
||||
}
|
||||
#endif // McTemplateU0xx_def
|
||||
|
||||
#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION
|
||||
|
||||
#if defined(__cplusplus)
|
||||
}
|
||||
#endif
|
||||
Binary file not shown.
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,17 @@
|
||||
## Primitives:
|
||||
|
||||
- `prim.c` contains Windows primitives for OS allocation.
|
||||
|
||||
## Event Tracing for Windows (ETW)
|
||||
|
||||
- `etw.h` is generated from `etw.man` which contains the manifest for mimalloc events.
|
||||
(100 is an allocation, 101 is for a free)
|
||||
|
||||
- `etw-mimalloc.wprp` is a profile for the Windows Performance Recorder (WPR).
|
||||
In an admin prompt, you can use:
|
||||
```
|
||||
> wpr -start src\prim\windows\etw-mimalloc.wprp -filemode
|
||||
> <my mimalloc program>
|
||||
> wpr -stop test.etl
|
||||
```
|
||||
and then open `test.etl` in the Windows Performance Analyzer (WPA).
|
||||
@@ -0,0 +1,258 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2019-2021, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc/internal.h"
|
||||
#include "mimalloc/prim.h" // _mi_prim_random_buf
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
We use our own PRNG to keep predictable performance of random number generation
|
||||
and to avoid implementations that use a lock. We only use the OS provided
|
||||
random source to initialize the initial seeds. Since we do not need ultimate
|
||||
performance but we do rely on the security (for secret cookies in secure mode)
|
||||
we use a cryptographically secure generator (chacha20).
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
#define MI_CHACHA_ROUNDS (20) // perhaps use 12 for better performance?
|
||||
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
Chacha20 implementation as the original algorithm with a 64-bit nonce
|
||||
and counter: https://en.wikipedia.org/wiki/Salsa20
|
||||
The input matrix has sixteen 32-bit values:
|
||||
Position 0 to 3: constant key
|
||||
Position 4 to 11: the key
|
||||
Position 12 to 13: the counter.
|
||||
Position 14 to 15: the nonce.
|
||||
|
||||
The implementation uses regular C code which compiles very well on modern compilers.
|
||||
(gcc x64 has no register spills, and clang 6+ uses SSE instructions)
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
static inline void qround(uint32_t x[16], size_t a, size_t b, size_t c, size_t d) {
|
||||
x[a] += x[b]; x[d] = mi_rotl32(x[d] ^ x[a], 16);
|
||||
x[c] += x[d]; x[b] = mi_rotl32(x[b] ^ x[c], 12);
|
||||
x[a] += x[b]; x[d] = mi_rotl32(x[d] ^ x[a], 8);
|
||||
x[c] += x[d]; x[b] = mi_rotl32(x[b] ^ x[c], 7);
|
||||
}
|
||||
|
||||
static void chacha_block(mi_random_ctx_t* ctx)
|
||||
{
|
||||
// scramble into `x`
|
||||
uint32_t x[16];
|
||||
for (size_t i = 0; i < 16; i++) {
|
||||
x[i] = ctx->input[i];
|
||||
}
|
||||
for (size_t i = 0; i < MI_CHACHA_ROUNDS; i += 2) {
|
||||
qround(x, 0, 4, 8, 12);
|
||||
qround(x, 1, 5, 9, 13);
|
||||
qround(x, 2, 6, 10, 14);
|
||||
qround(x, 3, 7, 11, 15);
|
||||
qround(x, 0, 5, 10, 15);
|
||||
qround(x, 1, 6, 11, 12);
|
||||
qround(x, 2, 7, 8, 13);
|
||||
qround(x, 3, 4, 9, 14);
|
||||
}
|
||||
|
||||
// add scrambled data to the initial state
|
||||
for (size_t i = 0; i < 16; i++) {
|
||||
ctx->output[i] = x[i] + ctx->input[i];
|
||||
}
|
||||
ctx->output_available = 16;
|
||||
|
||||
// increment the counter for the next round
|
||||
ctx->input[12] += 1;
|
||||
if (ctx->input[12] == 0) {
|
||||
ctx->input[13] += 1;
|
||||
if (ctx->input[13] == 0) { // and keep increasing into the nonce
|
||||
ctx->input[14] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static uint32_t chacha_next32(mi_random_ctx_t* ctx) {
|
||||
if (ctx->output_available <= 0) {
|
||||
chacha_block(ctx);
|
||||
ctx->output_available = 16; // (assign again to suppress static analysis warning)
|
||||
}
|
||||
const uint32_t x = ctx->output[16 - ctx->output_available];
|
||||
ctx->output[16 - ctx->output_available] = 0; // reset once the data is handed out
|
||||
ctx->output_available--;
|
||||
return x;
|
||||
}
|
||||
|
||||
static inline uint32_t read32(const uint8_t* p, size_t idx32) {
|
||||
const size_t i = 4*idx32;
|
||||
return ((uint32_t)p[i+0] | (uint32_t)p[i+1] << 8 | (uint32_t)p[i+2] << 16 | (uint32_t)p[i+3] << 24);
|
||||
}
|
||||
|
||||
static void chacha_init(mi_random_ctx_t* ctx, const uint8_t key[32], uint64_t nonce)
|
||||
{
|
||||
// since we only use chacha for randomness (and not encryption) we
|
||||
// do not _need_ to read 32-bit values as little endian but we do anyways
|
||||
// just for being compatible :-)
|
||||
ctx->output_available = 0;
|
||||
_mi_memzero(ctx->output,sizeof(ctx->output));
|
||||
for (size_t i = 0; i < 4; i++) {
|
||||
const uint8_t* sigma = (uint8_t*)"expand 32-byte k";
|
||||
ctx->input[i] = read32(sigma,i);
|
||||
}
|
||||
for (size_t i = 0; i < 8; i++) {
|
||||
ctx->input[i + 4] = read32(key,i);
|
||||
}
|
||||
ctx->input[12] = 0;
|
||||
ctx->input[13] = 0;
|
||||
ctx->input[14] = (uint32_t)nonce;
|
||||
ctx->input[15] = (uint32_t)(nonce >> 32);
|
||||
}
|
||||
|
||||
static void chacha_split(mi_random_ctx_t* ctx, uint64_t nonce, mi_random_ctx_t* ctx_new) {
|
||||
_mi_memzero(ctx_new, sizeof(*ctx_new));
|
||||
ctx_new->weak = ctx->weak;
|
||||
_mi_memcpy(ctx_new->input, ctx->input, sizeof(ctx_new->input));
|
||||
ctx_new->input[12] = 0;
|
||||
ctx_new->input[13] = 0;
|
||||
ctx_new->input[14] = (uint32_t)nonce;
|
||||
ctx_new->input[15] = (uint32_t)(nonce >> 32);
|
||||
mi_assert_internal(ctx->input[14] != ctx_new->input[14] || ctx->input[15] != ctx_new->input[15]); // do not reuse nonces!
|
||||
chacha_block(ctx_new);
|
||||
}
|
||||
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
Random interface
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
#if MI_DEBUG>1
|
||||
static bool mi_random_is_initialized(mi_random_ctx_t* ctx) {
|
||||
return (ctx != NULL && ctx->input[0] != 0);
|
||||
}
|
||||
#endif
|
||||
|
||||
void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* ctx_new) {
|
||||
mi_assert_internal(mi_random_is_initialized(ctx));
|
||||
mi_assert_internal(ctx != ctx_new);
|
||||
chacha_split(ctx, (uintptr_t)ctx_new /*nonce*/, ctx_new);
|
||||
}
|
||||
|
||||
uintptr_t _mi_random_next(mi_random_ctx_t* ctx) {
|
||||
mi_assert_internal(mi_random_is_initialized(ctx));
|
||||
uintptr_t r;
|
||||
do {
|
||||
#if MI_INTPTR_SIZE <= 4
|
||||
r = chacha_next32(ctx);
|
||||
#elif MI_INTPTR_SIZE == 8
|
||||
r = (((uintptr_t)chacha_next32(ctx) << 32) | chacha_next32(ctx));
|
||||
#else
|
||||
# error "define mi_random_next for this platform"
|
||||
#endif
|
||||
} while (r==0);
|
||||
return r;
|
||||
}
|
||||
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
To initialize a fresh random context.
|
||||
If we cannot get good randomness, we fall back to weak randomness based on a timer and ASLR.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
uintptr_t _mi_os_random_weak(uintptr_t extra_seed) {
|
||||
uintptr_t x = (uintptr_t)&_mi_os_random_weak ^ extra_seed; // ASLR makes the address random
|
||||
x ^= _mi_prim_clock_now();
|
||||
// and do a few randomization steps
|
||||
uintptr_t max = ((x ^ (x >> 17)) & 0x0F) + 1;
|
||||
for (uintptr_t i = 0; i < max || x==0; i++, x++) {
|
||||
x = _mi_random_shuffle(x);
|
||||
}
|
||||
mi_assert_internal(x != 0);
|
||||
return x;
|
||||
}
|
||||
|
||||
static void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) {
|
||||
uint8_t key[32];
|
||||
if (use_weak || !_mi_prim_random_buf(key, sizeof(key))) {
|
||||
// if we fail to get random data from the OS, we fall back to a
|
||||
// weak random source based on the current time
|
||||
#if !defined(__wasi__)
|
||||
if (!use_weak) { _mi_warning_message("unable to use secure randomness\n"); }
|
||||
#endif
|
||||
uintptr_t x = _mi_os_random_weak(0);
|
||||
for (size_t i = 0; i < 32; i+=4, x++) {
|
||||
x = _mi_random_shuffle(x);
|
||||
key[i] = (uint8_t)(x);
|
||||
key[i+1] = (uint8_t)(x>>8);
|
||||
key[i+2] = (uint8_t)(x>>16);
|
||||
key[i+3] = (uint8_t)(x>>24);
|
||||
}
|
||||
ctx->weak = true;
|
||||
}
|
||||
else {
|
||||
ctx->weak = false;
|
||||
}
|
||||
chacha_init(ctx, key, (uintptr_t)ctx /*nonce*/ );
|
||||
}
|
||||
|
||||
void _mi_random_init(mi_random_ctx_t* ctx) {
|
||||
mi_random_init_ex(ctx, false);
|
||||
}
|
||||
|
||||
void _mi_random_init_weak(mi_random_ctx_t * ctx) {
|
||||
mi_random_init_ex(ctx, true);
|
||||
}
|
||||
|
||||
void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx) {
|
||||
if (ctx->weak) {
|
||||
_mi_random_init(ctx);
|
||||
}
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------
|
||||
test vectors from <https://tools.ietf.org/html/rfc8439>
|
||||
----------------------------------------------------------- */
|
||||
/*
|
||||
static bool array_equals(uint32_t* x, uint32_t* y, size_t n) {
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
if (x[i] != y[i]) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
static void chacha_test(void)
|
||||
{
|
||||
uint32_t x[4] = { 0x11111111, 0x01020304, 0x9b8d6f43, 0x01234567 };
|
||||
uint32_t x_out[4] = { 0xea2a92f4, 0xcb1cf8ce, 0x4581472e, 0x5881c4bb };
|
||||
qround(x, 0, 1, 2, 3);
|
||||
mi_assert_internal(array_equals(x, x_out, 4));
|
||||
|
||||
uint32_t y[16] = {
|
||||
0x879531e0, 0xc5ecf37d, 0x516461b1, 0xc9a62f8a,
|
||||
0x44c20ef3, 0x3390af7f, 0xd9fc690b, 0x2a5f714c,
|
||||
0x53372767, 0xb00a5631, 0x974c541a, 0x359e9963,
|
||||
0x5c971061, 0x3d631689, 0x2098d9d6, 0x91dbd320 };
|
||||
uint32_t y_out[16] = {
|
||||
0x879531e0, 0xc5ecf37d, 0xbdb886dc, 0xc9a62f8a,
|
||||
0x44c20ef3, 0x3390af7f, 0xd9fc690b, 0xcfacafd2,
|
||||
0xe46bea80, 0xb00a5631, 0x974c541a, 0x359e9963,
|
||||
0x5c971061, 0xccc07c79, 0x2098d9d6, 0x91dbd320 };
|
||||
qround(y, 2, 7, 8, 13);
|
||||
mi_assert_internal(array_equals(y, y_out, 16));
|
||||
|
||||
mi_random_ctx_t r = {
|
||||
{ 0x61707865, 0x3320646e, 0x79622d32, 0x6b206574,
|
||||
0x03020100, 0x07060504, 0x0b0a0908, 0x0f0e0d0c,
|
||||
0x13121110, 0x17161514, 0x1b1a1918, 0x1f1e1d1c,
|
||||
0x00000001, 0x09000000, 0x4a000000, 0x00000000 },
|
||||
{0},
|
||||
0
|
||||
};
|
||||
uint32_t r_out[16] = {
|
||||
0xe4e7f110, 0x15593bd1, 0x1fdd0f50, 0xc47120a3,
|
||||
0xc7f4d1c7, 0x0368c033, 0x9aaa2204, 0x4e6cd4c3,
|
||||
0x466482d2, 0x09aa9f07, 0x05d7c214, 0xa2028bd9,
|
||||
0xd19c12b5, 0xb94e16de, 0xe883d0cb, 0x4e3c50a2 };
|
||||
chacha_block(&r);
|
||||
mi_assert_internal(array_equals(r.output, r_out, 16));
|
||||
}
|
||||
*/
|
||||
@@ -0,0 +1,43 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2020, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#ifndef _DEFAULT_SOURCE
|
||||
#define _DEFAULT_SOURCE
|
||||
#endif
|
||||
#if defined(__sun)
|
||||
// same remarks as os.c for the static's context.
|
||||
#undef _XOPEN_SOURCE
|
||||
#undef _POSIX_C_SOURCE
|
||||
#endif
|
||||
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc/internal.h"
|
||||
|
||||
// For a static override we create a single object file
|
||||
// containing the whole library. If it is linked first
|
||||
// it will override all the standard library allocation
|
||||
// functions (on Unix's).
|
||||
#include "alloc.c" // includes alloc-override.c and free.c
|
||||
#include "alloc-aligned.c"
|
||||
#include "alloc-posix.c"
|
||||
#include "arena.c"
|
||||
#include "arena-meta.c"
|
||||
#include "bitmap.c"
|
||||
#include "heap.c"
|
||||
#include "init.c"
|
||||
#include "libc.c"
|
||||
#include "options.c"
|
||||
#include "os.c"
|
||||
#include "page.c" // includes page-queue.c
|
||||
#include "page-map.c"
|
||||
#include "random.c"
|
||||
#include "stats.c"
|
||||
#include "theap.c"
|
||||
#include "threadlocal.c"
|
||||
#include "prim/prim.c"
|
||||
#if MI_OSX_ZONE
|
||||
#include "prim/osx/alloc-override-zone.c"
|
||||
#endif
|
||||
@@ -0,0 +1,813 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc-stats.h"
|
||||
#include "mimalloc/internal.h"
|
||||
#include "mimalloc/atomic.h"
|
||||
#include "mimalloc/prim.h"
|
||||
|
||||
#include <string.h> // memset
|
||||
|
||||
#if defined(_MSC_VER) && (_MSC_VER < 1920)
|
||||
#pragma warning(disable:4204) // non-constant aggregate initializer
|
||||
#endif
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Statistics operations
|
||||
----------------------------------------------------------- */
|
||||
|
||||
static void mi_stat_update_mt(mi_stat_count_t* stat, int64_t amount) {
|
||||
if (amount == 0) return;
|
||||
// add atomically
|
||||
int64_t current = mi_atomic_addi64_relaxed(&stat->current, amount);
|
||||
mi_atomic_maxi64_relaxed(&stat->peak, current + amount);
|
||||
if (amount > 0) {
|
||||
mi_atomic_addi64_relaxed(&stat->total, amount);
|
||||
}
|
||||
}
|
||||
|
||||
static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) {
|
||||
if (amount == 0) return;
|
||||
// add thread local
|
||||
stat->current += amount;
|
||||
if (stat->current > stat->peak) { stat->peak = stat->current; }
|
||||
if (amount > 0) { stat->total += amount; }
|
||||
}
|
||||
|
||||
|
||||
void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount) {
|
||||
mi_atomic_addi64_relaxed(&stat->total, (int64_t)amount);
|
||||
}
|
||||
|
||||
void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) {
|
||||
stat->total += amount;
|
||||
}
|
||||
|
||||
void __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount) {
|
||||
mi_stat_update_mt(stat, (int64_t)amount);
|
||||
}
|
||||
void __mi_stat_increase(mi_stat_count_t* stat, size_t amount) {
|
||||
mi_stat_update(stat, (int64_t)amount);
|
||||
}
|
||||
|
||||
void __mi_stat_decrease_mt(mi_stat_count_t* stat, size_t amount) {
|
||||
mi_stat_update_mt(stat, -((int64_t)amount));
|
||||
}
|
||||
void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount) {
|
||||
mi_stat_update(stat, -((int64_t)amount));
|
||||
}
|
||||
|
||||
|
||||
// Adjust stats to compensate; for example before committing a range,
|
||||
// first adjust downwards with parts that were already committed so
|
||||
// we avoid double counting.
|
||||
static void mi_stat_adjust_mt(mi_stat_count_t* stat, int64_t amount) {
|
||||
if (amount == 0) return;
|
||||
// adjust atomically
|
||||
mi_atomic_addi64_relaxed(&stat->current, amount);
|
||||
mi_atomic_addi64_relaxed(&stat->total, amount);
|
||||
}
|
||||
|
||||
static void mi_stat_adjust(mi_stat_count_t* stat, int64_t amount) {
|
||||
if (amount == 0) return;
|
||||
stat->current += amount;
|
||||
stat->total += amount;
|
||||
}
|
||||
|
||||
void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount) {
|
||||
mi_stat_adjust_mt(stat, (int64_t)amount);
|
||||
}
|
||||
void __mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount) {
|
||||
mi_stat_adjust(stat, (int64_t)amount);
|
||||
}
|
||||
void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount) {
|
||||
mi_stat_adjust_mt(stat, -((int64_t)amount));
|
||||
}
|
||||
void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount) {
|
||||
mi_stat_adjust(stat, -((int64_t)amount));
|
||||
}
|
||||
|
||||
|
||||
// must be thread safe as it is called from stats_merge
|
||||
static void mi_stat_count_add_mt(mi_stat_count_t* stat, const mi_stat_count_t* src) {
|
||||
if (stat==src) return;
|
||||
mi_atomic_void_addi64_relaxed(&stat->total, &src->total);
|
||||
const int64_t prev_current = mi_atomic_addi64_relaxed(&stat->current, src->current);
|
||||
|
||||
// Global current plus thread peak approximates new global peak
|
||||
// note: peak scores do really not work across threads.
|
||||
// we used to just add them together but that often overestimates in practice.
|
||||
// similarly, max does not seem to work well. The current approach
|
||||
// by Artem Kharytoniuk (@artem-lunarg) seems to work better, see PR#1112
|
||||
// for a longer description.
|
||||
mi_atomic_maxi64_relaxed(&stat->peak, prev_current + src->peak);
|
||||
}
|
||||
|
||||
static void mi_stat_counter_add_mt(mi_stat_counter_t* stat, const mi_stat_counter_t* src) {
|
||||
if (stat==src) return;
|
||||
mi_atomic_void_addi64_relaxed(&stat->total, &src->total);
|
||||
}
|
||||
|
||||
#define MI_STAT_COUNT(stat) mi_stat_count_add_mt(&stats->stat, &src->stat);
|
||||
#define MI_STAT_COUNTER(stat) mi_stat_counter_add_mt(&stats->stat, &src->stat);
|
||||
|
||||
// must be thread safe as it is called from stats_merge
|
||||
static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) {
|
||||
if (stats==NULL || src==NULL || stats==src) return;
|
||||
|
||||
// copy all fields
|
||||
MI_STAT_FIELDS()
|
||||
|
||||
#if MI_STAT>1
|
||||
for (size_t i = 0; i <= MI_BIN_HUGE; i++) {
|
||||
mi_stat_count_add_mt(&stats->malloc_bins[i], &src->malloc_bins[i]);
|
||||
}
|
||||
#endif
|
||||
for (size_t i = 0; i <= MI_BIN_HUGE; i++) {
|
||||
mi_stat_count_add_mt(&stats->page_bins[i], &src->page_bins[i]);
|
||||
}
|
||||
}
|
||||
|
||||
#undef MI_STAT_COUNT
|
||||
#undef MI_STAT_COUNTER
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Display statistics
|
||||
----------------------------------------------------------- */
|
||||
|
||||
// unit > 0 : size in binary bytes
|
||||
// unit == 0: count as decimal
|
||||
// unit < 0 : count in binary
|
||||
static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, const char* fmt) {
|
||||
char buf[32]; _mi_memzero_var(buf);
|
||||
int len = 32;
|
||||
const char* suffix = (unit <= 0 ? " " : "B");
|
||||
const int64_t base = (unit == 0 ? 1000 : 1024);
|
||||
if (unit>0) n *= unit;
|
||||
|
||||
const int64_t pos = (n < 0 ? -n : n);
|
||||
if (pos < base) {
|
||||
if (n!=1 || suffix[0] != 'B') { // skip printing 1 B for the unit column
|
||||
_mi_snprintf(buf, len, "%lld %-3s", (long long)n, (n==0 ? "" : suffix));
|
||||
}
|
||||
}
|
||||
else {
|
||||
int64_t divider = base;
|
||||
const char* magnitude = "K";
|
||||
if (pos >= divider*base) { divider *= base; magnitude = "M"; }
|
||||
if (pos >= divider*base) { divider *= base; magnitude = "G"; }
|
||||
const int64_t tens = (n / (divider/10));
|
||||
const long whole = (long)(tens/10);
|
||||
const long frac1 = (long)(tens%10);
|
||||
char unitdesc[8];
|
||||
_mi_snprintf(unitdesc, 8, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix);
|
||||
_mi_snprintf(buf, len, "%ld.%ld %-3s", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc);
|
||||
}
|
||||
_mi_fprintf(out, arg, (fmt==NULL ? "%12s" : fmt), buf);
|
||||
}
|
||||
|
||||
|
||||
static void mi_print_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg) {
|
||||
mi_printf_amount(n,unit,out,arg,NULL);
|
||||
}
|
||||
|
||||
static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out, void* arg) {
|
||||
if (unit==1) _mi_fprintf(out, arg, "%12s"," ");
|
||||
else mi_print_amount(n,0,out,arg);
|
||||
}
|
||||
|
||||
static void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg, const char* notok ) {
|
||||
_mi_fprintf(out, arg," %-10s:", msg);
|
||||
if (unit != 0) {
|
||||
if (unit > 0) {
|
||||
mi_print_amount(stat->peak, unit, out, arg);
|
||||
mi_print_amount(stat->total, unit, out, arg);
|
||||
// mi_print_amount(stat->freed, unit, out, arg);
|
||||
mi_print_amount(stat->current, unit, out, arg);
|
||||
mi_print_amount(unit, 1, out, arg);
|
||||
mi_print_count(stat->total, unit, out, arg);
|
||||
}
|
||||
else {
|
||||
mi_print_amount(stat->peak, -1, out, arg);
|
||||
mi_print_amount(stat->total, -1, out, arg);
|
||||
// mi_print_amount(stat->freed, -1, out, arg);
|
||||
mi_print_amount(stat->current, -1, out, arg);
|
||||
if (unit == -1) {
|
||||
_mi_fprintf(out, arg, "%24s", "");
|
||||
}
|
||||
else {
|
||||
mi_print_amount(-unit, 1, out, arg);
|
||||
mi_print_count((stat->total / -unit), 0, out, arg);
|
||||
}
|
||||
}
|
||||
if (stat->current != 0) {
|
||||
_mi_fprintf(out, arg, " ");
|
||||
_mi_fprintf(out, arg, (notok == NULL ? "not all freed" : notok));
|
||||
_mi_fprintf(out, arg, "\n");
|
||||
}
|
||||
else {
|
||||
_mi_fprintf(out, arg, " ok\n");
|
||||
}
|
||||
}
|
||||
else {
|
||||
mi_print_amount(stat->peak, 0, out, arg);
|
||||
mi_print_amount(stat->total, 0, out, arg);
|
||||
mi_print_amount(stat->current, 0, out, arg);
|
||||
_mi_fprintf(out, arg, "\n");
|
||||
}
|
||||
}
|
||||
|
||||
static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) {
|
||||
mi_stat_print_ex(stat, msg, unit, out, arg, NULL);
|
||||
}
|
||||
|
||||
#if MI_STAT>1
|
||||
static void mi_stat_total_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) {
|
||||
_mi_fprintf(out, arg, " %-10s:", msg);
|
||||
_mi_fprintf(out, arg, "%12s", " "); // no peak
|
||||
mi_print_amount(stat->total, unit, out, arg);
|
||||
_mi_fprintf(out, arg, "\n");
|
||||
}
|
||||
#endif
|
||||
|
||||
static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) {
|
||||
_mi_fprintf(out, arg, " %-10s:", msg);
|
||||
mi_print_amount(stat->total, 0, out, arg);
|
||||
_mi_fprintf(out, arg, "\n");
|
||||
}
|
||||
|
||||
static void mi_stat_counter_print_size(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) {
|
||||
_mi_fprintf(out, arg, " %-10s:", msg);
|
||||
mi_print_amount(stat->total, 1, out, arg);
|
||||
_mi_fprintf(out, arg, "\n");
|
||||
}
|
||||
|
||||
static void mi_stat_average_print(int64_t count, int64_t total, const char* msg, mi_output_fun* out, void* arg) {
|
||||
const int64_t avg_tens = (count == 0 ? 0 : (total*10 / count));
|
||||
const int64_t avg_whole = avg_tens/10;
|
||||
const int64_t avg_frac1 = avg_tens%10;
|
||||
_mi_fprintf(out, arg, " %-10s: %5lld.%lld avg\n", msg, avg_whole, avg_frac1);
|
||||
}
|
||||
|
||||
|
||||
static void mi_print_header(const char* name,mi_output_fun* out, void* arg ) {
|
||||
_mi_fprintf(out, arg, " %-11s %11s %11s %11s %11s %11s\n",
|
||||
name, "peak ", "total ", "current ", "block ", "total# ");
|
||||
}
|
||||
|
||||
#if MI_STAT>1
|
||||
static bool mi_stats_print_bins(const mi_stat_count_t* bins, size_t max, mi_output_fun* out, void* arg) {
|
||||
bool found = false;
|
||||
char buf[64];
|
||||
for (size_t i = 0; i <= max; i++) {
|
||||
if (bins[i].total > 0) {
|
||||
found = true;
|
||||
const size_t unit = _mi_bin_size((uint8_t)i);
|
||||
const char* pagekind = (unit <= MI_SMALL_MAX_OBJ_SIZE ? "S" :
|
||||
(unit <= MI_MEDIUM_MAX_OBJ_SIZE ? "M" :
|
||||
(unit <= MI_LARGE_MAX_OBJ_SIZE ? "L" : "H")));
|
||||
_mi_snprintf(buf, 64, "bin%2s %3lu", pagekind, (long)i);
|
||||
mi_stat_print(&bins[i], buf, (int64_t)unit, out, arg);
|
||||
}
|
||||
}
|
||||
if (found) {
|
||||
_mi_fprintf(out, arg, "\n");
|
||||
}
|
||||
return found;
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
//------------------------------------------------------------
|
||||
// Use an output wrapper for line-buffered output
|
||||
// (which is nice when using loggers etc.)
|
||||
//------------------------------------------------------------
|
||||
typedef struct buffered_s {
|
||||
mi_output_fun* out; // original output function
|
||||
void* arg; // and state
|
||||
char* buf; // local buffer of at least size `count+1`
|
||||
size_t used; // currently used chars `used <= count`
|
||||
size_t count; // total chars available for output
|
||||
} buffered_t;
|
||||
|
||||
static void mi_buffered_flush(buffered_t* buf) {
|
||||
buf->buf[buf->used] = 0;
|
||||
_mi_fputs(buf->out, buf->arg, NULL, buf->buf);
|
||||
buf->used = 0;
|
||||
}
|
||||
|
||||
static void mi_cdecl mi_buffered_out(const char* msg, void* arg) {
|
||||
buffered_t* buf = (buffered_t*)arg;
|
||||
if (msg==NULL || buf==NULL) return;
|
||||
for (const char* src = msg; *src != 0; src++) {
|
||||
char c = *src;
|
||||
if (buf->used >= buf->count) mi_buffered_flush(buf);
|
||||
mi_assert_internal(buf->used < buf->count);
|
||||
buf->buf[buf->used++] = c;
|
||||
if (c == '\n') mi_buffered_flush(buf);
|
||||
}
|
||||
}
|
||||
|
||||
//------------------------------------------------------------
|
||||
// Print statistics
|
||||
//------------------------------------------------------------
|
||||
|
||||
mi_decl_export void mi_process_info_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept
|
||||
{
|
||||
size_t elapsed;
|
||||
size_t user_time;
|
||||
size_t sys_time;
|
||||
size_t current_rss;
|
||||
size_t peak_rss;
|
||||
size_t current_commit;
|
||||
size_t peak_commit;
|
||||
size_t page_faults;
|
||||
mi_process_info(&elapsed, &user_time, &sys_time, ¤t_rss, &peak_rss, ¤t_commit, &peak_commit, &page_faults);
|
||||
_mi_fprintf(out, arg, " %-10s: %5zu.%03zu s\n", "elapsed", elapsed/1000, elapsed%1000);
|
||||
_mi_fprintf(out, arg, " %-10s: user: %zu.%03zu s, system: %zu.%03zu s, faults: %zu, peak rss: ", "process",
|
||||
user_time/1000, user_time%1000, sys_time/1000, sys_time%1000, page_faults);
|
||||
mi_printf_amount((int64_t)peak_rss, 1, out, arg, "%s");
|
||||
if (peak_commit > 0) {
|
||||
_mi_fprintf(out, arg, ", peak commit: ");
|
||||
mi_printf_amount((int64_t)peak_commit, 1, out, arg, "%s");
|
||||
}
|
||||
_mi_fprintf(out, arg, "\n");
|
||||
}
|
||||
|
||||
void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_attr_noexcept {
|
||||
// wrap the output function to be line buffered
|
||||
char buf[256]; _mi_memzero_var(buf);
|
||||
buffered_t buffer = { out0, arg0, NULL, 0, 255 };
|
||||
buffer.buf = buf;
|
||||
mi_output_fun* out = &mi_buffered_out;
|
||||
void* arg = &buffer;
|
||||
|
||||
// and print using that
|
||||
_mi_fprintf(out, arg, "%s %zu\n", name, id);
|
||||
|
||||
if (stats->malloc_normal.total + stats->malloc_huge.total != 0) {
|
||||
#if MI_STAT>1
|
||||
mi_print_header("blocks", out, arg);
|
||||
mi_stats_print_bins(stats->malloc_bins, MI_BIN_HUGE, out, arg);
|
||||
#endif
|
||||
#if MI_STAT
|
||||
mi_stat_print(&stats->malloc_normal, "binned", (stats->malloc_normal_count.total == 0 ? -1 : 1), out, arg);
|
||||
mi_stat_print(&stats->malloc_huge, "huge", (stats->malloc_huge_count.total == 0 ? -1 : 1), out, arg);
|
||||
mi_stat_count_t total = { 0,0,0 };
|
||||
mi_stat_count_add_mt(&total, &stats->malloc_normal);
|
||||
mi_stat_count_add_mt(&total, &stats->malloc_huge);
|
||||
mi_stat_print_ex(&total, "total", 1, out, arg, "");
|
||||
#if MI_STAT>1
|
||||
mi_stat_total_print(&stats->malloc_requested, "malloc req", 1, out, arg);
|
||||
#endif
|
||||
_mi_fprintf(out, arg, "\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
if (stats->pages.total != 0) {
|
||||
mi_print_header("pages", out, arg);
|
||||
mi_stat_print_ex(&stats->page_committed, "touched", 1, out, arg, "");
|
||||
// mi_stat_print(&stats->segments, "segments", -1, out, arg);
|
||||
// mi_stat_print(&stats->segments_abandoned, "-abandoned", -1, out, arg);
|
||||
// mi_stat_print(&stats->segments_cache, "-cached", -1, out, arg);
|
||||
mi_stat_print(&stats->pages, "pages", 0, out, arg);
|
||||
mi_stat_print(&stats->pages_abandoned, "abandoned", 0, out, arg);
|
||||
mi_stat_counter_print(&stats->pages_reclaim_on_alloc, "reclaima", out, arg);
|
||||
mi_stat_counter_print(&stats->pages_reclaim_on_free, "reclaimf", out, arg);
|
||||
mi_stat_counter_print(&stats->pages_reabandon_full, "reabandon", out, arg);
|
||||
mi_stat_counter_print(&stats->pages_unabandon_busy_wait, "waits", out, arg);
|
||||
mi_stat_counter_print(&stats->pages_extended, "extended", out, arg);
|
||||
mi_stat_counter_print(&stats->pages_retire, "retire", out, arg);
|
||||
mi_stat_average_print(stats->page_searches_count.total, stats->page_searches.total, "searches", out, arg);
|
||||
_mi_fprintf(out, arg, "\n");
|
||||
}
|
||||
|
||||
if (stats->arena_count.total > 0) {
|
||||
mi_print_header("arenas", out, arg);
|
||||
mi_stat_print_ex(&stats->reserved, "reserved", 1, out, arg, "");
|
||||
mi_stat_print_ex(&stats->committed, "committed", 1, out, arg, "");
|
||||
mi_stat_counter_print_size(&stats->reset, "reset", out, arg);
|
||||
mi_stat_counter_print_size(&stats->purged, "purged", out, arg);
|
||||
|
||||
mi_stat_counter_print(&stats->arena_count, "arenas", out, arg);
|
||||
mi_stat_counter_print(&stats->arena_rollback_count, "rollback", out, arg);
|
||||
mi_stat_counter_print(&stats->mmap_calls, "mmaps", out, arg);
|
||||
mi_stat_counter_print(&stats->commit_calls, "commits", out, arg);
|
||||
mi_stat_counter_print(&stats->reset_calls, "resets", out, arg);
|
||||
mi_stat_counter_print(&stats->purge_calls, "purges", out, arg);
|
||||
mi_stat_counter_print(&stats->malloc_guarded_count, "guarded", out, arg);
|
||||
mi_stat_print_ex(&stats->theaps, "theaps", 0, out, arg, "");
|
||||
mi_stat_print_ex(&stats->heaps, "heaps", 0, out, arg, "");
|
||||
mi_stat_counter_print(&stats->heaps_delete_wait, "heap waits", out, arg);
|
||||
_mi_fprintf(out, arg, "\n");
|
||||
|
||||
mi_print_header("process", out, arg);
|
||||
mi_stat_print_ex(&stats->threads, "threads", 0, out, arg, "");
|
||||
_mi_fprintf(out, arg, " %-10s: %5i\n", "numa nodes", _mi_os_numa_node_count());
|
||||
mi_process_info_print_out(out, arg);
|
||||
}
|
||||
_mi_fprintf(out, arg, "\n");
|
||||
}
|
||||
|
||||
|
||||
static mi_msecs_t mi_process_start; // = 0
|
||||
|
||||
// called on process init
|
||||
void _mi_stats_init(void) {
|
||||
if (mi_process_start == 0) { mi_process_start = _mi_clock_start(); };
|
||||
}
|
||||
|
||||
static void mi_stats_add_into(mi_stats_t* to, const mi_stats_t* from) {
|
||||
mi_assert_internal(to != NULL && from != NULL);
|
||||
if (to == from) return;
|
||||
mi_stats_add(to, from);
|
||||
}
|
||||
|
||||
void _mi_stats_merge_into(mi_stats_t* to, mi_stats_t* from) {
|
||||
mi_assert_internal(to != NULL && from != NULL);
|
||||
if (to == from) return;
|
||||
mi_stats_add(to, from);
|
||||
_mi_memzero(from, sizeof(mi_stats_t));
|
||||
}
|
||||
|
||||
static const mi_stats_t* mi_stats_merge_theap_to_heap(mi_theap_t* theap) mi_attr_noexcept {
|
||||
mi_stats_t* stats = &theap->stats;
|
||||
mi_stats_t* heap_stats = &_mi_theap_heap(theap)->stats;
|
||||
_mi_stats_merge_into( heap_stats, stats );
|
||||
return heap_stats;
|
||||
}
|
||||
|
||||
static const mi_stats_t* mi_heap_get_stats(mi_heap_t* heap) {
|
||||
if (heap==NULL) { heap = mi_heap_main(); }
|
||||
mi_theap_t* theap = _mi_heap_theap_peek(heap);
|
||||
if (theap==NULL) return &heap->stats;
|
||||
else return mi_stats_merge_theap_to_heap(theap);
|
||||
}
|
||||
|
||||
// deprecated
|
||||
void mi_stats_reset(void) mi_attr_noexcept {
|
||||
if (!mi_theap_is_initialized(_mi_theap_default())) return;
|
||||
mi_heap_get_stats(mi_heap_main());
|
||||
mi_heap_stats_merge_to_subproc(mi_heap_main());
|
||||
}
|
||||
|
||||
|
||||
void mi_heap_stats_print_out(mi_heap_t* heap, mi_output_fun* out, void* arg) mi_attr_noexcept {
|
||||
if (heap==NULL) { heap = mi_heap_main(); }
|
||||
_mi_stats_print("heap", heap->heap_seq, mi_heap_get_stats(heap), out, arg);
|
||||
}
|
||||
|
||||
typedef struct mi_heap_print_visit_info_s {
|
||||
mi_output_fun* out;
|
||||
void* out_arg;
|
||||
} mi_heap_print_visit_info_t;
|
||||
|
||||
static bool mi_cdecl mi_heap_print_visitor(mi_heap_t* heap, void* arg) {
|
||||
mi_heap_print_visit_info_t* vinfo = (mi_heap_print_visit_info_t*)(arg);
|
||||
mi_heap_stats_print_out(heap, vinfo->out, vinfo->out_arg);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
// show each heap and then the subproc
|
||||
void mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg) mi_attr_noexcept {
|
||||
mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
|
||||
if (subproc==NULL) return;
|
||||
mi_heap_print_visit_info_t vinfo = { out, arg };
|
||||
mi_subproc_visit_heaps(subproc_id, &mi_heap_print_visitor, &vinfo);
|
||||
_mi_stats_print("subproc", subproc->subproc_seq, &subproc->stats, out, arg);
|
||||
}
|
||||
|
||||
|
||||
// aggregate all stats from the heaps and subproc and print those
|
||||
void mi_subproc_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg) mi_attr_noexcept {
|
||||
mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
|
||||
if (subproc==NULL) return;
|
||||
mi_stats_t_decl(stats);
|
||||
if (mi_subproc_stats_get(subproc_id, &stats)) {
|
||||
_mi_stats_print("subproc", subproc->subproc_seq, &stats, out, arg);
|
||||
}
|
||||
}
|
||||
|
||||
void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept {
|
||||
mi_subproc_stats_print_out(mi_subproc_current(),out, arg);
|
||||
}
|
||||
|
||||
// deprecated
|
||||
void mi_stats_print(void* out) mi_attr_noexcept {
|
||||
// for compatibility there is an `out` parameter (which can be `stdout` or `stderr`)
|
||||
mi_stats_print_out((mi_output_fun*)out, NULL);
|
||||
}
|
||||
|
||||
// deprecated
|
||||
void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept {
|
||||
mi_theap_t* theap = _mi_theap_default();
|
||||
if (theap==NULL || !mi_theap_is_initialized(theap)) return;
|
||||
_mi_stats_print("heap", _mi_theap_heap(theap)->heap_seq, &theap->stats, out, arg);
|
||||
mi_stats_merge_theap_to_heap(_mi_theap_default());
|
||||
}
|
||||
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// Basic timer for convenience; use milli-seconds to avoid doubles
|
||||
// ----------------------------------------------------------------
|
||||
|
||||
static mi_msecs_t mi_clock_diff;
|
||||
|
||||
mi_msecs_t _mi_clock_now(void) {
|
||||
return _mi_prim_clock_now();
|
||||
}
|
||||
|
||||
mi_msecs_t _mi_clock_start(void) {
|
||||
if (mi_clock_diff == 0.0) {
|
||||
mi_msecs_t t0 = _mi_clock_now();
|
||||
mi_clock_diff = _mi_clock_now() - t0;
|
||||
}
|
||||
return _mi_clock_now();
|
||||
}
|
||||
|
||||
mi_msecs_t _mi_clock_end(mi_msecs_t start) {
|
||||
mi_msecs_t end = _mi_clock_now();
|
||||
return (end - start - mi_clock_diff);
|
||||
}
|
||||
|
||||
|
||||
// --------------------------------------------------------
|
||||
// Basic process statistics
|
||||
// --------------------------------------------------------
|
||||
|
||||
mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept
|
||||
{
|
||||
mi_process_info_t pinfo;
|
||||
_mi_memzero_var(pinfo);
|
||||
pinfo.elapsed = _mi_clock_end(mi_process_start);
|
||||
{ const mi_subproc_t* subproc = _mi_subproc_main();
|
||||
if (subproc!=NULL) {
|
||||
pinfo.current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.current)));
|
||||
pinfo.peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.peak)));
|
||||
}
|
||||
}
|
||||
pinfo.current_rss = pinfo.current_commit;
|
||||
pinfo.peak_rss = pinfo.peak_commit;
|
||||
pinfo.utime = 0;
|
||||
pinfo.stime = 0;
|
||||
pinfo.page_faults = 0;
|
||||
|
||||
_mi_prim_process_info(&pinfo);
|
||||
|
||||
if (elapsed_msecs!=NULL) *elapsed_msecs = (pinfo.elapsed < 0 ? 0 : (pinfo.elapsed < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.elapsed : PTRDIFF_MAX));
|
||||
if (user_msecs!=NULL) *user_msecs = (pinfo.utime < 0 ? 0 : (pinfo.utime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.utime : PTRDIFF_MAX));
|
||||
if (system_msecs!=NULL) *system_msecs = (pinfo.stime < 0 ? 0 : (pinfo.stime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.stime : PTRDIFF_MAX));
|
||||
if (current_rss!=NULL) *current_rss = pinfo.current_rss;
|
||||
if (peak_rss!=NULL) *peak_rss = pinfo.peak_rss;
|
||||
if (current_commit!=NULL) *current_commit = pinfo.current_commit;
|
||||
if (peak_commit!=NULL) *peak_commit = pinfo.peak_commit;
|
||||
if (page_faults!=NULL) *page_faults = pinfo.page_faults;
|
||||
}
|
||||
|
||||
mi_decl_export void mi_process_info_print(void) mi_attr_noexcept {
|
||||
mi_process_info_print_out(NULL, NULL);
|
||||
}
|
||||
|
||||
|
||||
// --------------------------------------------------------
|
||||
// Return statistics
|
||||
// --------------------------------------------------------
|
||||
|
||||
size_t mi_stats_get_bin_size(size_t bin) mi_attr_noexcept {
|
||||
if (bin > MI_BIN_HUGE) return 0;
|
||||
return _mi_bin_size(bin);
|
||||
}
|
||||
|
||||
static bool mi_stats_copy(mi_stats_t* stats_to, const mi_stats_t* stats_from) mi_attr_noexcept {
|
||||
if (stats_to == NULL || stats_to->size != sizeof(mi_stats_t) || stats_to->version != MI_STAT_VERSION) return false;
|
||||
if (stats_from == NULL || stats_from->size != stats_to->size) return false;
|
||||
_mi_memcpy(stats_to, stats_from, stats_to->size);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool mi_subproc_stats_get_exclusive(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept {
|
||||
const mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
|
||||
if (subproc==NULL) return false;
|
||||
return mi_stats_copy(stats,&subproc->stats);
|
||||
}
|
||||
|
||||
bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats) mi_attr_noexcept {
|
||||
return mi_stats_copy(stats, mi_heap_get_stats(heap));
|
||||
}
|
||||
|
||||
|
||||
static bool mi_cdecl mi_heap_aggregate_visitor(mi_heap_t* heap, void* arg) {
|
||||
mi_stats_t* stats = (mi_stats_t*)arg;
|
||||
mi_stats_add_into(stats, mi_heap_get_stats(heap));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept {
|
||||
if (stats==NULL) return false;
|
||||
mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
|
||||
if (subproc == NULL) return false;
|
||||
if (!mi_stats_copy(stats, &subproc->stats)) return false;
|
||||
mi_subproc_visit_heaps(subproc_id, &mi_heap_aggregate_visitor, stats);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool mi_stats_get(mi_stats_t* stats) mi_attr_noexcept {
|
||||
return mi_subproc_stats_get(mi_subproc_current(), stats);
|
||||
}
|
||||
|
||||
|
||||
// --------------------------------------------------------
|
||||
// Statics in json format
|
||||
// --------------------------------------------------------
|
||||
|
||||
typedef struct mi_json_buf_s {
|
||||
char* buf;
|
||||
size_t size;
|
||||
size_t used;
|
||||
bool can_realloc;
|
||||
} mi_json_buf_t;
|
||||
|
||||
static bool mi_json_buf_expand(mi_json_buf_t* hbuf) {
|
||||
if (hbuf==NULL) return false;
|
||||
if (hbuf->buf != NULL && hbuf->size>0) {
|
||||
hbuf->buf[hbuf->size-1] = 0;
|
||||
}
|
||||
if (hbuf->size > SIZE_MAX/2 || !hbuf->can_realloc) return false;
|
||||
const size_t newsize = (hbuf->size == 0 ? mi_good_size(12*MI_KiB) : 2*hbuf->size);
|
||||
char* const newbuf = (char*)mi_rezalloc(hbuf->buf, newsize);
|
||||
if (newbuf == NULL) return false;
|
||||
hbuf->buf = newbuf;
|
||||
hbuf->size = newsize;
|
||||
return true;
|
||||
}
|
||||
|
||||
static void mi_json_buf_print(mi_json_buf_t* hbuf, const char* msg) {
|
||||
if (msg==NULL || hbuf==NULL) return;
|
||||
if (hbuf->used + 1 >= hbuf->size && !hbuf->can_realloc) return;
|
||||
for (const char* src = msg; *src != 0; src++) {
|
||||
char c = *src;
|
||||
if (hbuf->used + 1 >= hbuf->size) {
|
||||
if (!mi_json_buf_expand(hbuf)) return;
|
||||
}
|
||||
mi_assert_internal(hbuf->used < hbuf->size);
|
||||
hbuf->buf[hbuf->used++] = c;
|
||||
}
|
||||
mi_assert_internal(hbuf->used < hbuf->size);
|
||||
hbuf->buf[hbuf->used] = 0;
|
||||
}
|
||||
|
||||
static void mi_json_buf_print_count_bin(mi_json_buf_t* hbuf, const char* prefix, const mi_stat_count_t* stat, size_t bin, bool add_comma) {
|
||||
const size_t binsize = mi_stats_get_bin_size(bin);
|
||||
const size_t pagesize = (binsize <= MI_SMALL_MAX_OBJ_SIZE ? MI_SMALL_PAGE_SIZE :
|
||||
(binsize <= MI_MEDIUM_MAX_OBJ_SIZE ? MI_MEDIUM_PAGE_SIZE :
|
||||
(binsize <= MI_LARGE_MAX_OBJ_SIZE ? MI_LARGE_PAGE_SIZE : 0)));
|
||||
char buf[128];
|
||||
_mi_snprintf(buf, 128, "%s{ \"total\": %lld, \"peak\": %lld, \"current\": %lld, \"block_size\": %zu, \"page_size\": %zu }%s\n", prefix, stat->total, stat->peak, stat->current, binsize, pagesize, (add_comma ? "," : ""));
|
||||
buf[127] = 0;
|
||||
mi_json_buf_print(hbuf, buf);
|
||||
}
|
||||
|
||||
static void mi_json_buf_print_count_cbin(mi_json_buf_t* hbuf, const char* prefix, const mi_stat_count_t* stat, mi_chunkbin_t bin, bool add_comma) {
|
||||
const char* cbin = " ";
|
||||
switch(bin) {
|
||||
case MI_CBIN_SMALL: cbin = "S"; break;
|
||||
case MI_CBIN_MEDIUM: cbin = "M"; break;
|
||||
case MI_CBIN_LARGE: cbin = "L"; break;
|
||||
case MI_CBIN_HUGE: cbin = "H"; break;
|
||||
case MI_CBIN_OTHER: cbin = "X"; break;
|
||||
default: cbin = " "; break;
|
||||
}
|
||||
char buf[128];
|
||||
_mi_snprintf(buf, 128, "%s{ \"total\": %lld, \"peak\": %lld, \"current\": %lld, \"bin\": \"%s\" }%s\n", prefix, stat->total, stat->peak, stat->current, cbin, (add_comma ? "," : ""));
|
||||
buf[127] = 0;
|
||||
mi_json_buf_print(hbuf, buf);
|
||||
}
|
||||
|
||||
static void mi_json_buf_print_count(mi_json_buf_t* hbuf, const char* prefix, const mi_stat_count_t* stat, bool add_comma) {
|
||||
char buf[128];
|
||||
_mi_snprintf(buf, 128, "%s{ \"total\": %lld, \"peak\": %lld, \"current\": %lld }%s\n", prefix, stat->total, stat->peak, stat->current, (add_comma ? "," : ""));
|
||||
buf[127] = 0;
|
||||
mi_json_buf_print(hbuf, buf);
|
||||
}
|
||||
|
||||
static void mi_json_buf_print_count_value(mi_json_buf_t* hbuf, const char* name, const mi_stat_count_t* stat) {
|
||||
char buf[128];
|
||||
_mi_snprintf(buf, 128, " \"%s\": ", name);
|
||||
buf[127] = 0;
|
||||
mi_json_buf_print(hbuf, buf);
|
||||
mi_json_buf_print_count(hbuf, "", stat, true);
|
||||
}
|
||||
|
||||
static void mi_json_buf_print_value(mi_json_buf_t* hbuf, const char* name, int64_t val) {
|
||||
char buf[128];
|
||||
_mi_snprintf(buf, 128, " \"%s\": %lld,\n", name, val);
|
||||
buf[127] = 0;
|
||||
mi_json_buf_print(hbuf, buf);
|
||||
}
|
||||
|
||||
static void mi_json_buf_print_size(mi_json_buf_t* hbuf, const char* name, size_t val, bool add_comma) {
|
||||
char buf[128];
|
||||
_mi_snprintf(buf, 128, " \"%s\": %zu%s\n", name, val, (add_comma ? "," : ""));
|
||||
buf[127] = 0;
|
||||
mi_json_buf_print(hbuf, buf);
|
||||
}
|
||||
|
||||
static void mi_json_buf_print_counter_value(mi_json_buf_t* hbuf, const char* name, const mi_stat_counter_t* stat) {
|
||||
mi_json_buf_print_value(hbuf, name, stat->total);
|
||||
}
|
||||
|
||||
#define MI_STAT_COUNT(stat) mi_json_buf_print_count_value(&hbuf, #stat, &stats->stat);
|
||||
#define MI_STAT_COUNTER(stat) mi_json_buf_print_counter_value(&hbuf, #stat, &stats->stat);
|
||||
|
||||
static char* mi_stats_get_json_from(const mi_stats_t* stats, size_t output_size, char* output_buf) mi_attr_noexcept {
|
||||
if (stats==NULL || stats->size!=sizeof(mi_stats_t) || stats->version!=MI_STAT_VERSION) return NULL;
|
||||
mi_json_buf_t hbuf = { NULL, 0, 0, true };
|
||||
if (output_size > 0 && output_buf != NULL) {
|
||||
_mi_memzero(output_buf, output_size);
|
||||
hbuf.buf = output_buf;
|
||||
hbuf.size = output_size;
|
||||
hbuf.can_realloc = false;
|
||||
}
|
||||
else {
|
||||
if (!mi_json_buf_expand(&hbuf)) return NULL;
|
||||
}
|
||||
mi_json_buf_print(&hbuf, "{\n");
|
||||
mi_json_buf_print_value(&hbuf, "stat_version", MI_STAT_VERSION);
|
||||
mi_json_buf_print_value(&hbuf, "mimalloc_version", MI_MALLOC_VERSION);
|
||||
|
||||
// process info
|
||||
mi_json_buf_print(&hbuf, " \"process\": {\n");
|
||||
size_t elapsed;
|
||||
size_t user_time;
|
||||
size_t sys_time;
|
||||
size_t current_rss;
|
||||
size_t peak_rss;
|
||||
size_t current_commit;
|
||||
size_t peak_commit;
|
||||
size_t page_faults;
|
||||
mi_process_info(&elapsed, &user_time, &sys_time, ¤t_rss, &peak_rss, ¤t_commit, &peak_commit, &page_faults);
|
||||
mi_json_buf_print_size(&hbuf, "elapsed_msecs", elapsed, true);
|
||||
mi_json_buf_print_size(&hbuf, "user_msecs", user_time, true);
|
||||
mi_json_buf_print_size(&hbuf, "system_msecs", sys_time, true);
|
||||
mi_json_buf_print_size(&hbuf, "page_faults", page_faults, true);
|
||||
mi_json_buf_print_size(&hbuf, "rss_current", current_rss, true);
|
||||
mi_json_buf_print_size(&hbuf, "rss_peak", peak_rss, true);
|
||||
mi_json_buf_print_size(&hbuf, "commit_current", current_commit, true);
|
||||
mi_json_buf_print_size(&hbuf, "commit_peak", peak_commit, false);
|
||||
mi_json_buf_print(&hbuf, " },\n");
|
||||
|
||||
// statistics
|
||||
MI_STAT_FIELDS()
|
||||
|
||||
// size bins
|
||||
mi_json_buf_print(&hbuf, " \"malloc_bins\": [\n");
|
||||
for (size_t i = 0; i <= MI_BIN_HUGE; i++) {
|
||||
mi_json_buf_print_count_bin(&hbuf, " ", &stats->malloc_bins[i], i, i!=MI_BIN_HUGE);
|
||||
}
|
||||
mi_json_buf_print(&hbuf, " ],\n");
|
||||
mi_json_buf_print(&hbuf, " \"page_bins\": [\n");
|
||||
for (size_t i = 0; i <= MI_BIN_HUGE; i++) {
|
||||
mi_json_buf_print_count_bin(&hbuf, " ", &stats->page_bins[i], i, i!=MI_BIN_HUGE);
|
||||
}
|
||||
mi_json_buf_print(&hbuf, " ],\n");
|
||||
mi_json_buf_print(&hbuf, " \"chunk_bins\": [\n");
|
||||
for (size_t i = 0; i < MI_CBIN_COUNT; i++) {
|
||||
mi_json_buf_print_count_cbin(&hbuf, " ", &stats->chunk_bins[i], (mi_chunkbin_t)i, i!=MI_CBIN_COUNT-1);
|
||||
}
|
||||
mi_json_buf_print(&hbuf, " ]\n");
|
||||
mi_json_buf_print(&hbuf, "}\n");
|
||||
if (hbuf.used >= hbuf.size) {
|
||||
// failed
|
||||
if (hbuf.can_realloc) { mi_free(hbuf.buf); }
|
||||
return NULL;
|
||||
}
|
||||
else {
|
||||
return hbuf.buf;
|
||||
}
|
||||
}
|
||||
|
||||
char* mi_subproc_stats_get_json(mi_subproc_id_t subproc_id, size_t buf_size, char* buf) mi_attr_noexcept {
|
||||
mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
|
||||
if (subproc==NULL) return NULL;
|
||||
mi_stats_t_decl(stats);
|
||||
if (!mi_subproc_stats_get(subproc_id,&stats)) return NULL;
|
||||
return mi_stats_get_json_from(&stats, buf_size, buf);
|
||||
}
|
||||
|
||||
char* mi_heap_stats_get_json(mi_heap_t* heap, size_t buf_size, char* buf) mi_attr_noexcept {
|
||||
return mi_stats_get_json_from(mi_heap_get_stats(heap), buf_size, buf);
|
||||
}
|
||||
|
||||
char* mi_stats_get_json(size_t buf_size, char* buf) mi_attr_noexcept {
|
||||
return mi_subproc_stats_get_json(mi_subproc_current(), buf_size, buf);
|
||||
}
|
||||
|
||||
char* mi_stats_as_json(mi_stats_t* stats, size_t buf_size, char* buf) mi_attr_noexcept {
|
||||
return mi_stats_get_json_from(stats, buf_size, buf);
|
||||
}
|
||||
@@ -0,0 +1,715 @@
|
||||
/*----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc/internal.h"
|
||||
#include "mimalloc/prim.h" // _mi_theap_default
|
||||
|
||||
#if defined(_MSC_VER) && (_MSC_VER < 1920)
|
||||
#pragma warning(disable:4204) // non-constant aggregate initializer
|
||||
#endif
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Helpers
|
||||
----------------------------------------------------------- */
|
||||
|
||||
// return `true` if ok, `false` to break
|
||||
typedef bool (theap_page_visitor_fun)(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2);
|
||||
|
||||
// Visit all pages in a theap; returns `false` if break was called.
|
||||
static bool mi_theap_visit_pages(mi_theap_t* theap, theap_page_visitor_fun* fn, bool include_full, void* arg1, void* arg2)
|
||||
{
|
||||
if (theap==NULL || theap->page_count==0) return 0;
|
||||
|
||||
// visit all pages
|
||||
#if MI_DEBUG>1
|
||||
size_t total = theap->page_count;
|
||||
size_t count = 0;
|
||||
#endif
|
||||
|
||||
const size_t max_bin = (include_full ? MI_BIN_FULL : MI_BIN_FULL - 1);
|
||||
for (size_t i = 0; i <= max_bin; i++) {
|
||||
mi_page_queue_t* pq = &theap->pages[i];
|
||||
mi_page_t* page = pq->first;
|
||||
while(page != NULL) {
|
||||
mi_page_t* next = page->next; // save next in case the page gets removed from the queue
|
||||
mi_assert_internal(mi_page_theap(page) == theap);
|
||||
#if MI_DEBUG>1
|
||||
count++;
|
||||
#endif
|
||||
if (!fn(theap, pq, page, arg1, arg2)) return false;
|
||||
page = next; // and continue
|
||||
}
|
||||
}
|
||||
mi_assert_internal(!include_full || count == total);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
#if MI_DEBUG>=2
|
||||
static bool mi_theap_page_is_valid(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) {
|
||||
MI_UNUSED(arg1);
|
||||
MI_UNUSED(arg2);
|
||||
MI_UNUSED(pq);
|
||||
mi_assert_internal(mi_page_theap(page) == theap);
|
||||
mi_assert_expensive(_mi_page_is_valid(page));
|
||||
return true;
|
||||
}
|
||||
#endif
|
||||
#if MI_DEBUG>=3
|
||||
static bool mi_theap_is_valid(mi_theap_t* theap) {
|
||||
mi_assert_internal(theap!=NULL);
|
||||
mi_theap_visit_pages(theap, &mi_theap_page_is_valid, true, NULL, NULL);
|
||||
for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) {
|
||||
mi_assert_internal(_mi_page_queue_is_valid(theap, &theap->pages[bin]));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
"Collect" pages by migrating `local_free` and `thread_free`
|
||||
lists and freeing empty pages. This is done when a thread
|
||||
stops (and in that case abandons pages if there are still
|
||||
blocks alive)
|
||||
----------------------------------------------------------- */
|
||||
|
||||
typedef enum mi_collect_e {
|
||||
MI_NORMAL,
|
||||
MI_FORCE,
|
||||
MI_ABANDON
|
||||
} mi_collect_t;
|
||||
|
||||
|
||||
static bool mi_theap_page_collect(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg_collect, void* arg2 ) {
|
||||
MI_UNUSED(arg2);
|
||||
MI_UNUSED(theap);
|
||||
mi_assert_internal(mi_theap_page_is_valid(theap, pq, page, NULL, NULL));
|
||||
mi_collect_t collect = *((mi_collect_t*)arg_collect);
|
||||
_mi_page_free_collect(page, collect >= MI_FORCE);
|
||||
if (mi_page_all_free(page)) {
|
||||
// no more used blocks, possibly free the page.
|
||||
if (collect >= MI_FORCE || page->retire_expire == 0) { // either forced/abandon, or not already retired
|
||||
// note: this will potentially free retired pages as well.
|
||||
_mi_page_free(page, pq);
|
||||
}
|
||||
}
|
||||
else if (collect == MI_ABANDON) {
|
||||
// still used blocks but the thread is done; abandon the page
|
||||
_mi_page_abandon(page, pq);
|
||||
}
|
||||
return true; // don't break
|
||||
}
|
||||
|
||||
static void mi_theap_merge_stats(mi_theap_t* theap) {
|
||||
mi_assert_internal(mi_theap_is_initialized(theap));
|
||||
_mi_stats_merge_into(&_mi_theap_heap(theap)->stats, &theap->stats);
|
||||
}
|
||||
|
||||
static void mi_theap_collect_ex(mi_theap_t* theap, mi_collect_t collect)
|
||||
{
|
||||
if (theap==NULL || !mi_theap_is_initialized(theap)) return;
|
||||
mi_assert_expensive(mi_theap_is_valid(theap));
|
||||
|
||||
const bool force = (collect >= MI_FORCE);
|
||||
_mi_deferred_free(theap, force);
|
||||
|
||||
// python/cpython#112532: we may be called from a thread that is not the owner of the theap
|
||||
// const bool is_main_thread = (_mi_is_main_thread() && theap->thread_id == _mi_thread_id());
|
||||
|
||||
// collect retired pages
|
||||
_mi_theap_collect_retired(theap, force);
|
||||
|
||||
// collect all pages owned by this thread
|
||||
mi_theap_visit_pages(theap, &mi_theap_page_collect, (collect!=MI_NORMAL), &collect, NULL); // dont normally visit full pages, see issue #1220
|
||||
|
||||
// collect arenas (this is program wide so don't force purges on abandonment of threads)
|
||||
//mi_atomic_storei64_release(&theap->tld->subproc->purge_expire, 1);
|
||||
_mi_arenas_collect(collect == MI_FORCE /* force purge? */, collect >= MI_FORCE /* visit all? */, theap->tld);
|
||||
|
||||
// merge statistics
|
||||
mi_theap_merge_stats(theap);
|
||||
}
|
||||
|
||||
void _mi_theap_collect_abandon(mi_theap_t* theap) {
|
||||
mi_theap_collect_ex(theap, MI_ABANDON);
|
||||
}
|
||||
|
||||
void mi_theap_collect(mi_theap_t* theap, bool force) mi_attr_noexcept {
|
||||
mi_theap_collect_ex(theap, (force ? MI_FORCE : MI_NORMAL));
|
||||
}
|
||||
|
||||
void mi_collect(bool force) mi_attr_noexcept {
|
||||
// cannot really collect process wide, just a theap..
|
||||
mi_theap_collect(_mi_theap_default(), force);
|
||||
}
|
||||
|
||||
void mi_heap_collect(mi_heap_t* heap, bool force) {
|
||||
// cannot really collect a heap, just a theap..
|
||||
mi_theap_collect(mi_heap_theap(heap), force);
|
||||
}
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Heap new
|
||||
----------------------------------------------------------- */
|
||||
|
||||
mi_theap_t* mi_theap_get_default(void) {
|
||||
mi_theap_t* theap = _mi_theap_default();
|
||||
if mi_unlikely(!mi_theap_is_initialized(theap)) {
|
||||
mi_thread_init();
|
||||
theap = _mi_theap_default();
|
||||
mi_assert_internal(mi_theap_is_initialized(theap));
|
||||
}
|
||||
return theap;
|
||||
}
|
||||
|
||||
// todo: make order of parameters consistent (but would that break compat with CPython?)
|
||||
void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld)
|
||||
{
|
||||
mi_assert_internal(theap!=NULL);
|
||||
mi_assert_internal(heap!=NULL);
|
||||
mi_memid_t memid = theap->memid;
|
||||
_mi_memcpy_aligned(theap, &_mi_theap_empty, sizeof(mi_theap_t));
|
||||
theap->memid = memid;
|
||||
theap->refcount = 1;
|
||||
theap->tld = tld; // avoid reading the thread-local tld during initialization
|
||||
mi_atomic_store_ptr_relaxed(mi_heap_t,&theap->heap,heap);
|
||||
mi_assert_internal(theap->stats.size == sizeof(mi_stats_t));
|
||||
|
||||
_mi_theap_options_init(theap);
|
||||
if (theap->tld->is_in_threadpool) {
|
||||
// if we run as part of a thread pool it is better to not arbitrarily reclaim abandoned pages into our theap.
|
||||
// this is checked in `free.c:mi_free_try_collect_mt`
|
||||
// .. but abandoning is good in this case: halve the full page retain (possibly to 0)
|
||||
// (so blocked threads do not hold on to too much memory)
|
||||
if (theap->page_full_retain > 0) {
|
||||
theap->page_full_retain = theap->page_full_retain / 4;
|
||||
}
|
||||
}
|
||||
|
||||
// push on the thread local theaps list
|
||||
mi_theap_t* head = NULL;
|
||||
mi_lock(&theap->tld->theaps_lock) {
|
||||
head = theap->tld->theaps;
|
||||
theap->tprev = NULL;
|
||||
theap->tnext = head;
|
||||
if (head!=NULL) { head->tprev = theap; }
|
||||
theap->tld->theaps = theap;
|
||||
}
|
||||
|
||||
// initialize random
|
||||
if (head == NULL) { // first theap in this thread?
|
||||
#if defined(_WIN32) && !defined(MI_SHARED_LIB)
|
||||
_mi_random_init_weak(&theap->random); // prevent allocation failure during bcrypt dll initialization with static linking (issue #1185)
|
||||
#else
|
||||
_mi_random_init(&theap->random);
|
||||
#endif
|
||||
}
|
||||
else {
|
||||
_mi_random_split(&head->random, &theap->random);
|
||||
}
|
||||
theap->cookie = _mi_theap_random_next(theap) | 1;
|
||||
_mi_theap_guarded_init(theap);
|
||||
mi_subproc_stat_increase(_mi_subproc(),theaps,1);
|
||||
|
||||
// push on the heap's theap list
|
||||
mi_lock(&heap->theaps_lock) {
|
||||
head = heap->theaps;
|
||||
theap->hprev = NULL;
|
||||
theap->hnext = head;
|
||||
if (head!=NULL) { head->hprev = theap; }
|
||||
heap->theaps = theap;
|
||||
}
|
||||
}
|
||||
|
||||
mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld) {
|
||||
mi_assert_internal(tld!=NULL);
|
||||
mi_assert_internal(heap!=NULL);
|
||||
// allocate and initialize a theap
|
||||
mi_memid_t memid;
|
||||
mi_theap_t* theap;
|
||||
//if (!_mi_is_heap_main(heap)) {
|
||||
// theap = (mi_theap_t*)mi_heap_zalloc(mi_heap_main(),sizeof(mi_theap_t));
|
||||
// memid = _mi_memid_create(MI_MEM_HEAP_MAIN);
|
||||
// memid.initially_zero = memid.initially_committed = true;
|
||||
//}
|
||||
//else
|
||||
if (heap->exclusive_arena == NULL) {
|
||||
theap = (mi_theap_t*)_mi_meta_zalloc(sizeof(mi_theap_t), &memid);
|
||||
}
|
||||
else {
|
||||
// theaps associated with a specific arena are allocated in that arena
|
||||
// note: takes up at least one slice which is quite wasteful...
|
||||
const size_t size = _mi_align_up(sizeof(mi_theap_t),MI_ARENA_MIN_OBJ_SIZE);
|
||||
theap = (mi_theap_t*)_mi_arenas_alloc(heap, size, true, true, heap->exclusive_arena, tld->thread_seq, tld->numa_node, &memid);
|
||||
mi_assert_internal(memid.mem.os.size >= size);
|
||||
}
|
||||
if (theap==NULL) {
|
||||
_mi_error_message(ENOMEM, "unable to allocate theap meta-data\n");
|
||||
return NULL;
|
||||
}
|
||||
theap->memid = memid;
|
||||
_mi_theap_init(theap, heap, tld);
|
||||
return theap;
|
||||
}
|
||||
|
||||
uintptr_t _mi_theap_random_next(mi_theap_t* theap) {
|
||||
return _mi_random_next(&theap->random);
|
||||
}
|
||||
|
||||
static void mi_theap_free_mem(mi_theap_t* theap) {
|
||||
if (theap!=NULL) {
|
||||
mi_subproc_stat_decrease(_mi_subproc(),theaps,1);
|
||||
// free the used memory
|
||||
if (theap->memid.memkind == MI_MEM_HEAP_MAIN) { // note: for now unused as it would access theap_default stats in mi_free of the current theap
|
||||
mi_assert_internal(_mi_is_heap_main(mi_heap_of(theap)));
|
||||
mi_free(theap);
|
||||
}
|
||||
else if (theap->memid.memkind == MI_MEM_META) {
|
||||
_mi_meta_free(theap, sizeof(*theap), theap->memid);
|
||||
}
|
||||
else {
|
||||
_mi_arenas_free(theap, _mi_align_up(sizeof(*theap),MI_ARENA_MIN_OBJ_SIZE), theap->memid ); // issue #1168, avoid assertion failure
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void _mi_theap_incref(mi_theap_t* theap) {
|
||||
if (theap!=NULL && theap->memid.memkind > MI_MEM_STATIC) {
|
||||
mi_atomic_increment_acq_rel(&theap->refcount);
|
||||
}
|
||||
}
|
||||
|
||||
void _mi_theap_decref(mi_theap_t* theap) {
|
||||
if (theap!=NULL && theap->memid.memkind > MI_MEM_STATIC) {
|
||||
if (mi_atomic_decrement_acq_rel(&theap->refcount) == 1) {
|
||||
mi_theap_free_mem(theap);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// called from `mi_theap_delete` to free the internal theap resources.
|
||||
bool _mi_theap_free(mi_theap_t* theap, bool acquire_heap_theaps_lock, bool acquire_tld_theaps_lock) {
|
||||
mi_assert(theap != NULL);
|
||||
if (theap==NULL) return true;
|
||||
|
||||
mi_heap_t* const heap = mi_atomic_exchange_ptr_acq_rel(mi_heap_t, &theap->heap, NULL);
|
||||
if (heap==NULL) {
|
||||
// concurrent interaction, retry in an outer loop (as the other thread may be blocked on our lock)
|
||||
return false;
|
||||
}
|
||||
else {
|
||||
// merge stats to the owning heap
|
||||
_mi_stats_merge_into(&heap->stats, &theap->stats);
|
||||
|
||||
// remove ourselves from the heap theaps list
|
||||
mi_lock_maybe(&heap->theaps_lock, acquire_heap_theaps_lock) {
|
||||
if (theap->hnext != NULL) { theap->hnext->hprev = theap->hprev; }
|
||||
if (theap->hprev != NULL) { theap->hprev->hnext = theap->hnext; }
|
||||
else { mi_assert_internal(heap->theaps == theap); heap->theaps = theap->hnext; }
|
||||
theap->hnext = theap->hprev = NULL;
|
||||
}
|
||||
|
||||
// remove ourselves from the thread local theaps list
|
||||
mi_lock_maybe(&theap->tld->theaps_lock, acquire_tld_theaps_lock) {
|
||||
if (theap->tnext != NULL) { theap->tnext->tprev = theap->tprev; }
|
||||
if (theap->tprev != NULL) { theap->tprev->tnext = theap->tnext; }
|
||||
else { mi_assert_internal(theap->tld->theaps == theap); theap->tld->theaps = theap->tnext; }
|
||||
theap->tnext = theap->tprev = NULL;
|
||||
}
|
||||
theap->tld = NULL;
|
||||
_mi_theap_decref(theap);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Heap destroy
|
||||
----------------------------------------------------------- */
|
||||
/*
|
||||
|
||||
// zero out the page queues
|
||||
static void mi_theap_reset_pages(mi_theap_t* theap) {
|
||||
mi_assert_internal(theap != NULL);
|
||||
mi_assert_internal(mi_theap_is_initialized(theap));
|
||||
// TODO: copy full empty theap instead?
|
||||
_mi_memset(&theap->pages_free_direct, 0, sizeof(theap->pages_free_direct));
|
||||
_mi_memcpy_aligned(&theap->pages, &_mi_theap_empty.pages, sizeof(theap->pages));
|
||||
// theap->thread_delayed_free = NULL;
|
||||
theap->page_count = 0;
|
||||
}
|
||||
|
||||
static bool _mi_theap_page_destroy(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) {
|
||||
MI_UNUSED(arg1);
|
||||
MI_UNUSED(arg2);
|
||||
MI_UNUSED(pq);
|
||||
|
||||
// ensure no more thread_delayed_free will be added
|
||||
//_mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE, false);
|
||||
|
||||
// stats
|
||||
const size_t bsize = mi_page_block_size(page);
|
||||
if (bsize > MI_LARGE_MAX_OBJ_SIZE) {
|
||||
mi_theap_stat_decrease(theap, malloc_huge, bsize);
|
||||
}
|
||||
#if (MI_STAT>0)
|
||||
_mi_page_free_collect(page, false); // update used count
|
||||
const size_t inuse = page->used;
|
||||
if (bsize <= MI_LARGE_MAX_OBJ_SIZE) {
|
||||
mi_theap_stat_decrease(theap, malloc_normal, bsize * inuse);
|
||||
#if (MI_STAT>1)
|
||||
mi_theap_stat_decrease(theap, malloc_bins[_mi_bin(bsize)], inuse);
|
||||
#endif
|
||||
}
|
||||
// mi_theap_stat_decrease(theap, malloc_requested, bsize * inuse); // todo: off for aligned blocks...
|
||||
#endif
|
||||
|
||||
/// pretend it is all free now
|
||||
mi_assert_internal(mi_page_thread_free(page) == NULL);
|
||||
page->used = 0;
|
||||
|
||||
// and free the page
|
||||
// mi_page_free(page,false);
|
||||
page->next = NULL;
|
||||
page->prev = NULL;
|
||||
mi_page_set_theap(page, NULL);
|
||||
_mi_arenas_page_free(page, theap);
|
||||
|
||||
return true; // keep going
|
||||
}
|
||||
|
||||
void _mi_theap_destroy_pages(mi_theap_t* theap) {
|
||||
mi_theap_visit_pages(theap, &_mi_theap_page_destroy, NULL, NULL);
|
||||
mi_theap_reset_pages(theap);
|
||||
}
|
||||
|
||||
#if MI_TRACK_HEAP_DESTROY
|
||||
static bool mi_cdecl mi_theap_track_block_free(const mi_theap_t* theap, const mi_theap_area_t* area, void* block, size_t block_size, void* arg) {
|
||||
MI_UNUSED(theap); MI_UNUSED(area); MI_UNUSED(arg); MI_UNUSED(block_size);
|
||||
mi_track_free_size(block,mi_usable_size(block));
|
||||
return true;
|
||||
}
|
||||
#endif
|
||||
|
||||
void mi_theap_destroy(mi_theap_t* theap) {
|
||||
mi_assert(theap != NULL);
|
||||
mi_assert(mi_theap_is_initialized(theap));
|
||||
mi_assert(!theap->allow_page_reclaim);
|
||||
mi_assert(!theap->allow_page_abandon);
|
||||
mi_assert_expensive(mi_theap_is_valid(theap));
|
||||
if (theap==NULL || !mi_theap_is_initialized(theap)) return;
|
||||
#if MI_GUARDED
|
||||
// _mi_warning_message("'mi_theap_destroy' called but MI_GUARDED is enabled -- using `mi_theap_delete` instead (theap at %p)\n", theap);
|
||||
mi_theap_delete(theap);
|
||||
return;
|
||||
#else
|
||||
if (theap->allow_page_reclaim) {
|
||||
_mi_warning_message("'mi_theap_destroy' called but ignored as the theap was not created with 'allow_destroy' (theap at %p)\n", theap);
|
||||
// don't free in case it may contain reclaimed pages,
|
||||
mi_theap_delete(theap);
|
||||
}
|
||||
else {
|
||||
// track all blocks as freed
|
||||
#if MI_TRACK_HEAP_DESTROY
|
||||
mi_theap_visit_blocks(theap, true, mi_theap_track_block_free, NULL);
|
||||
#endif
|
||||
// free all pages
|
||||
_mi_theap_destroy_pages(theap);
|
||||
mi_theap_free(theap,true);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
// forcefully destroy all theaps in the current thread
|
||||
void _mi_theap_unsafe_destroy_all(mi_theap_t* theap) {
|
||||
mi_assert_internal(theap != NULL);
|
||||
if (theap == NULL) return;
|
||||
mi_theap_t* curr = theap->tld->theaps;
|
||||
while (curr != NULL) {
|
||||
mi_theap_t* next = curr->next;
|
||||
if (!curr->allow_page_reclaim) {
|
||||
mi_theap_destroy(curr);
|
||||
}
|
||||
else {
|
||||
_mi_theap_destroy_pages(curr);
|
||||
}
|
||||
curr = next;
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Safe Heap delete
|
||||
----------------------------------------------------------- */
|
||||
|
||||
// Safe delete a theap without freeing any still allocated blocks in that theap.
|
||||
void _mi_theap_delete(mi_theap_t* theap, bool acquire_tld_theaps_lock)
|
||||
{
|
||||
mi_assert(theap != NULL);
|
||||
mi_assert(mi_theap_is_initialized(theap));
|
||||
mi_assert_expensive(mi_theap_is_valid(theap));
|
||||
if (theap==NULL || !mi_theap_is_initialized(theap)) return;
|
||||
|
||||
// abandon all pages
|
||||
_mi_theap_collect_abandon(theap);
|
||||
|
||||
mi_assert_internal(theap->page_count==0);
|
||||
_mi_theap_free(theap, true /* acquire heap->theaps_lock */, acquire_tld_theaps_lock);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Load/unload theaps
|
||||
----------------------------------------------------------- */
|
||||
/*
|
||||
void mi_theap_unload(mi_theap_t* theap) {
|
||||
mi_assert(mi_theap_is_initialized(theap));
|
||||
mi_assert_expensive(mi_theap_is_valid(theap));
|
||||
if (theap==NULL || !mi_theap_is_initialized(theap)) return;
|
||||
if (_mi_theap_heap(theap)->exclusive_arena == NULL) {
|
||||
_mi_warning_message("cannot unload theaps that are not associated with an exclusive arena\n");
|
||||
return;
|
||||
}
|
||||
|
||||
// abandon all pages so all thread'id in the pages are cleared
|
||||
_mi_theap_collect_abandon(theap);
|
||||
mi_assert_internal(theap->page_count==0);
|
||||
|
||||
// remove from theap list
|
||||
mi_theap_free(theap, false); // but don't actually free the memory
|
||||
|
||||
// disassociate from the current thread-local and static state
|
||||
theap->tld = NULL;
|
||||
return;
|
||||
}
|
||||
|
||||
bool mi_theap_reload(mi_theap_t* theap, mi_arena_id_t arena_id) {
|
||||
mi_assert(mi_theap_is_initialized(theap));
|
||||
if (theap==NULL || !mi_theap_is_initialized(theap)) return false;
|
||||
if (_mi_theap_heap(theap)->exclusive_arena == NULL) {
|
||||
_mi_warning_message("cannot reload theaps that were not associated with an exclusive arena\n");
|
||||
return false;
|
||||
}
|
||||
if (theap->tld != NULL) {
|
||||
_mi_warning_message("cannot reload theaps that were not unloaded first\n");
|
||||
return false;
|
||||
}
|
||||
mi_arena_t* arena = _mi_arena_from_id(arena_id);
|
||||
if (_mi_theap_heap(theap)->exclusive_arena != arena) {
|
||||
_mi_warning_message("trying to reload a theap at a different arena address: %p vs %p\n", _mi_theap_heap(theap)->exclusive_arena, arena);
|
||||
return false;
|
||||
}
|
||||
|
||||
mi_assert_internal(theap->page_count==0);
|
||||
|
||||
// re-associate with the current thread-local and static state
|
||||
theap->tld = mi_theap_get_default()->tld;
|
||||
|
||||
// reinit direct pages (as we may be in a different process)
|
||||
mi_assert_internal(theap->page_count == 0);
|
||||
for (size_t i = 0; i < MI_PAGES_DIRECT; i++) {
|
||||
theap->pages_free_direct[i] = (mi_page_t*)&_mi_page_empty;
|
||||
}
|
||||
|
||||
// push on the thread local theaps list
|
||||
theap->tnext = theap->tld->theaps;
|
||||
theap->tld->theaps = theap;
|
||||
return true;
|
||||
}
|
||||
*/
|
||||
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Visit all theap blocks and areas
|
||||
Todo: enable visiting abandoned pages, and
|
||||
enable visiting all blocks of all theaps across threads
|
||||
----------------------------------------------------------- */
|
||||
|
||||
void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) {
|
||||
const size_t bsize = mi_page_block_size(page);
|
||||
const size_t ubsize = mi_page_usable_block_size(page);
|
||||
area->reserved = page->reserved * bsize;
|
||||
area->committed = page->capacity * bsize;
|
||||
area->blocks = mi_page_start(page);
|
||||
area->used = page->used; // number of blocks in use (#553)
|
||||
area->block_size = ubsize;
|
||||
area->full_block_size = bsize;
|
||||
area->reserved1 = page;
|
||||
}
|
||||
|
||||
static void mi_get_fast_divisor(size_t divisor, uint64_t* magic, size_t* shift) {
|
||||
mi_assert_internal(divisor > 0 && divisor <= UINT32_MAX);
|
||||
*shift = MI_SIZE_BITS - mi_clz(divisor - 1);
|
||||
*magic = ((((uint64_t)1 << 32) * (((uint64_t)1 << *shift) - divisor)) / divisor + 1);
|
||||
}
|
||||
|
||||
static size_t mi_fast_divide(size_t n, uint64_t magic, size_t shift) {
|
||||
mi_assert_internal(n <= UINT32_MAX);
|
||||
const uint64_t hi = ((uint64_t)n * magic) >> 32;
|
||||
return (size_t)((hi + n) >> shift);
|
||||
}
|
||||
|
||||
bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg) {
|
||||
mi_assert(area != NULL);
|
||||
if (area==NULL) return true;
|
||||
mi_assert(page != NULL);
|
||||
if (page == NULL) return true;
|
||||
|
||||
_mi_page_free_collect(page,true); // collect both thread_delayed and local_free
|
||||
mi_assert_internal(page->local_free == NULL);
|
||||
if (page->used == 0) return true;
|
||||
|
||||
size_t psize;
|
||||
uint8_t* const pstart = mi_page_area(page, &psize);
|
||||
mi_heap_t* const heap = mi_page_heap(page);
|
||||
const size_t bsize = mi_page_block_size(page);
|
||||
const size_t ubsize = mi_page_usable_block_size(page); // without padding
|
||||
|
||||
// optimize page with one block
|
||||
if (page->capacity == 1) {
|
||||
mi_assert_internal(page->used == 1 && page->free == NULL);
|
||||
return visitor(heap, area, pstart, ubsize, arg);
|
||||
}
|
||||
mi_assert(bsize <= UINT32_MAX);
|
||||
|
||||
// optimize full pages
|
||||
if (page->used == page->capacity) {
|
||||
uint8_t* block = pstart;
|
||||
for (size_t i = 0; i < page->capacity; i++) {
|
||||
if (!visitor(heap, area, block, ubsize, arg)) return false;
|
||||
block += bsize;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// create a bitmap of free blocks.
|
||||
#define MI_MAX_BLOCKS (MI_SMALL_PAGE_SIZE / sizeof(void*))
|
||||
uintptr_t free_map[MI_MAX_BLOCKS / MI_INTPTR_BITS];
|
||||
const uintptr_t bmapsize = _mi_divide_up(page->capacity, MI_INTPTR_BITS);
|
||||
memset(free_map, 0, bmapsize * sizeof(intptr_t));
|
||||
if (page->capacity % MI_INTPTR_BITS != 0) {
|
||||
// mark left-over bits at the end as free
|
||||
size_t shift = (page->capacity % MI_INTPTR_BITS);
|
||||
uintptr_t mask = (UINTPTR_MAX << shift);
|
||||
free_map[bmapsize - 1] = mask;
|
||||
}
|
||||
|
||||
// fast repeated division by the block size
|
||||
uint64_t magic;
|
||||
size_t shift;
|
||||
mi_get_fast_divisor(bsize, &magic, &shift);
|
||||
|
||||
#if MI_DEBUG>1
|
||||
size_t free_count = 0;
|
||||
#endif
|
||||
for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page, block)) {
|
||||
#if MI_DEBUG>1
|
||||
free_count++;
|
||||
#endif
|
||||
mi_assert_internal((uint8_t*)block >= pstart && (uint8_t*)block < (pstart + psize));
|
||||
size_t offset = (uint8_t*)block - pstart;
|
||||
mi_assert_internal(offset % bsize == 0);
|
||||
mi_assert_internal(offset <= UINT32_MAX);
|
||||
size_t blockidx = mi_fast_divide(offset, magic, shift);
|
||||
mi_assert_internal(blockidx == offset / bsize);
|
||||
mi_assert_internal(blockidx < MI_MAX_BLOCKS);
|
||||
size_t bitidx = (blockidx / MI_INTPTR_BITS);
|
||||
size_t bit = blockidx - (bitidx * MI_INTPTR_BITS);
|
||||
free_map[bitidx] |= ((uintptr_t)1 << bit);
|
||||
}
|
||||
mi_assert_internal(page->capacity == (free_count + page->used));
|
||||
|
||||
// walk through all blocks skipping the free ones
|
||||
#if MI_DEBUG>1
|
||||
size_t used_count = 0;
|
||||
#endif
|
||||
uint8_t* block = pstart;
|
||||
for (size_t i = 0; i < bmapsize; i++) {
|
||||
if (free_map[i] == 0) {
|
||||
// every block is in use
|
||||
for (size_t j = 0; j < MI_INTPTR_BITS; j++) {
|
||||
#if MI_DEBUG>1
|
||||
used_count++;
|
||||
#endif
|
||||
if (!visitor(heap, area, block, ubsize, arg)) return false;
|
||||
block += bsize;
|
||||
}
|
||||
}
|
||||
else {
|
||||
// visit the used blocks in the mask
|
||||
uintptr_t m = ~free_map[i];
|
||||
while (m != 0) {
|
||||
#if MI_DEBUG>1
|
||||
used_count++;
|
||||
#endif
|
||||
size_t bitidx = mi_ctz(m);
|
||||
if (!visitor(heap, area, block + (bitidx * bsize), ubsize, arg)) return false;
|
||||
m &= m - 1; // clear least significant bit
|
||||
}
|
||||
block += bsize * MI_INTPTR_BITS;
|
||||
}
|
||||
}
|
||||
mi_assert_internal(page->used == used_count);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
|
||||
// Separate struct to keep `mi_page_t` out of the public interface
|
||||
typedef struct mi_theap_area_ex_s {
|
||||
mi_heap_area_t area;
|
||||
mi_page_t* page;
|
||||
} mi_theap_area_ex_t;
|
||||
|
||||
typedef bool (mi_theap_area_visit_fun)(const mi_theap_t* theap, const mi_theap_area_ex_t* area, void* arg);
|
||||
|
||||
static bool mi_theap_visit_areas_page(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* vfun, void* arg) {
|
||||
MI_UNUSED(theap);
|
||||
MI_UNUSED(pq);
|
||||
mi_theap_area_visit_fun* fun = (mi_theap_area_visit_fun*)vfun;
|
||||
mi_theap_area_ex_t xarea;
|
||||
xarea.page = page;
|
||||
_mi_heap_area_init(&xarea.area, page);
|
||||
return fun(theap, &xarea, arg);
|
||||
}
|
||||
|
||||
// Visit all theap pages as areas
|
||||
static bool mi_theap_visit_areas(const mi_theap_t* theap, mi_theap_area_visit_fun* visitor, void* arg) {
|
||||
if (visitor == NULL) return false;
|
||||
return mi_theap_visit_pages((mi_theap_t*)theap, &mi_theap_visit_areas_page, true, (void*)(visitor), arg); // note: function pointer to void* :-{
|
||||
}
|
||||
|
||||
// Just to pass arguments
|
||||
typedef struct mi_visit_blocks_args_s {
|
||||
bool visit_blocks;
|
||||
mi_block_visit_fun* visitor;
|
||||
void* arg;
|
||||
} mi_visit_blocks_args_t;
|
||||
|
||||
static bool mi_theap_area_visitor(const mi_theap_t* theap, const mi_theap_area_ex_t* xarea, void* arg) {
|
||||
mi_visit_blocks_args_t* args = (mi_visit_blocks_args_t*)arg;
|
||||
if (!args->visitor(_mi_theap_heap(theap), &xarea->area, NULL, xarea->area.block_size, args->arg)) return false;
|
||||
if (args->visit_blocks) {
|
||||
return _mi_theap_area_visit_blocks(&xarea->area, xarea->page, args->visitor, args->arg);
|
||||
}
|
||||
else {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Visit all blocks in a theap
|
||||
bool mi_theap_visit_blocks(const mi_theap_t* theap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) {
|
||||
mi_visit_blocks_args_t args = { visit_blocks, visitor, arg };
|
||||
return mi_theap_visit_areas(theap, &mi_theap_area_visitor, &args);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2019-2026, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
Implement dynamic thread local variables (for heap's).
|
||||
Unlike most OS native implementations there is no limit on the number
|
||||
that can be allocated.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc/internal.h"
|
||||
#include "mimalloc/prim.h"
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Each thread can have (a dynamically expanding) array of
|
||||
thread-local values. Each slot has a value and a version.
|
||||
The version is used to safely reuse slots.
|
||||
----------------------------------------------------------- */
|
||||
typedef struct mi_tls_slot_s {
|
||||
size_t version;
|
||||
void* value;
|
||||
} mi_tls_slot_t;
|
||||
|
||||
typedef struct mi_thread_locals_s {
|
||||
size_t count;
|
||||
mi_tls_slot_t slots[1];
|
||||
} mi_thread_locals_t;
|
||||
|
||||
static mi_thread_locals_t mi_thread_locals_empty = { 0, {{0,NULL}} };
|
||||
|
||||
mi_decl_thread mi_thread_locals_t* mi_thread_locals = &mi_thread_locals_empty; // always point to a valid `mi_thread_locals_t`
|
||||
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Each key consists of the slot index in the lower bits,
|
||||
and its version it the top bits. When we get a value
|
||||
the version must match or we return NULL. When we set
|
||||
a value, we also set the version of the key.
|
||||
----------------------------------------------------------- */
|
||||
|
||||
#if MI_SIZE_BITS < 64
|
||||
#define MI_TLS_IDX_BITS (MI_SIZE_BITS/2) // half for the index, half for the version
|
||||
#else
|
||||
#define MI_TLS_IDX_BITS (MI_SIZE_BITS/4) // 16 bits for the index, 48 bits for the version
|
||||
#endif
|
||||
#define MI_TLS_IDX_MASK ((MI_ZU(1)<<MI_TLS_IDX_BITS)-1)
|
||||
#define MI_TLS_IDX_MAX MI_TLS_IDX_MASK
|
||||
#define MI_TLS_VERSION_MAX ((MI_ZU(1)<<(MI_SIZE_BITS - MI_TLS_IDX_BITS))-1)
|
||||
|
||||
static size_t mi_key_index( size_t key ) {
|
||||
return (key & MI_TLS_IDX_MASK);
|
||||
}
|
||||
|
||||
static size_t mi_key_version( size_t key ) {
|
||||
return (key >> MI_TLS_IDX_BITS);
|
||||
}
|
||||
|
||||
static mi_thread_local_t mi_key_create( size_t index, size_t version ) {
|
||||
mi_assert_internal(version != 0 && version <= MI_TLS_VERSION_MAX);
|
||||
mi_assert_internal(index <= MI_TLS_IDX_MAX);
|
||||
const mi_thread_local_t key = ((version << MI_TLS_IDX_BITS) | index);
|
||||
mi_assert_internal(key != 0);
|
||||
return key;
|
||||
}
|
||||
|
||||
|
||||
// dynamically reallocate the thread local slots when needed
|
||||
static mi_thread_locals_t* mi_thread_locals_expand(size_t least_idx) {
|
||||
mi_thread_locals_t* tls_old = mi_thread_locals;
|
||||
const size_t count_old = tls_old->count;
|
||||
size_t count;
|
||||
if (count_old==0) {
|
||||
tls_old = NULL; // so we allocate fresh from mi_thread_locals_empty
|
||||
count = 16; // start with 16 slots
|
||||
}
|
||||
else if (count_old >= 1024) {
|
||||
count = count_old + 1024; // at some point increase linearly
|
||||
}
|
||||
else {
|
||||
count = 2*count_old; // and double initially
|
||||
}
|
||||
if (count <= least_idx) {
|
||||
count = least_idx + 1;
|
||||
}
|
||||
if (count > MI_TLS_IDX_MAX) { return NULL; } // too large
|
||||
mi_thread_locals_t* tls = (mi_thread_locals_t*)mi_rezalloc(tls_old, sizeof(mi_thread_locals_t) + count*sizeof(mi_tls_slot_t));
|
||||
if mi_unlikely(tls==NULL) return NULL;
|
||||
tls->count = count;
|
||||
mi_thread_locals = tls;
|
||||
return tls;
|
||||
}
|
||||
|
||||
static mi_decl_noinline bool mi_thread_local_set_expand( mi_thread_local_t key, void* val ) {
|
||||
if (val==NULL) return true;
|
||||
const size_t idx = mi_key_index(key);
|
||||
mi_thread_locals_t* tls = mi_thread_locals_expand(idx);
|
||||
if (tls==NULL) return false;
|
||||
mi_assert_internal(tls == mi_thread_locals);
|
||||
mi_assert_internal(idx < tls->count);
|
||||
tls->slots[idx].value = val;
|
||||
tls->slots[idx].version = mi_key_version(key);
|
||||
return true;
|
||||
}
|
||||
|
||||
// set a tls slot; returns `true` if successful.
|
||||
// Can return `false` if we could not reallocate the slots array.
|
||||
bool _mi_thread_local_set( mi_thread_local_t key, void* val ) {
|
||||
mi_thread_locals_t* tls = mi_thread_locals;
|
||||
mi_assert_internal(tls!=NULL);
|
||||
mi_assert_internal(key!=0);
|
||||
const size_t idx = mi_key_index(key);
|
||||
if mi_likely(idx < tls->count) {
|
||||
tls->slots[idx].value = val;
|
||||
tls->slots[idx].version = mi_key_version(key);
|
||||
return true;
|
||||
}
|
||||
else {
|
||||
return mi_thread_local_set_expand( key, val ); // tailcall
|
||||
}
|
||||
}
|
||||
|
||||
// get a tls slot value
|
||||
void* _mi_thread_local_get( mi_thread_local_t key ) {
|
||||
const mi_thread_locals_t* const tls = mi_thread_locals;
|
||||
mi_assert_internal(tls!=NULL);
|
||||
mi_assert_internal(key!=0);
|
||||
const size_t idx = mi_key_index(key);
|
||||
if mi_likely(idx < tls->count && mi_key_version(key) == tls->slots[idx].version) {
|
||||
return tls->slots[idx].value;
|
||||
}
|
||||
else {
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
void _mi_thread_locals_thread_done(void) {
|
||||
mi_thread_locals_t* const tls = mi_thread_locals;
|
||||
if (tls!=NULL && tls->count > 0) {
|
||||
mi_free(tls);
|
||||
mi_thread_locals = &mi_thread_locals_empty;
|
||||
}
|
||||
}
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Create and free fresh TLS key's
|
||||
----------------------------------------------------------- */
|
||||
#include "bitmap.h"
|
||||
|
||||
static mi_lock_t mi_thread_locals_lock; // we need a lock in order to re-allocate the slot bits
|
||||
static mi_bitmap_t* mi_thread_locals_free; // reuse an arena bitmap to track which slots were assigned (1=free, 0=in-use)
|
||||
static size_t mi_thread_locals_version; // version to be able to reuse slots safely
|
||||
|
||||
void _mi_thread_locals_init(void) {
|
||||
mi_lock_init(&mi_thread_locals_lock);
|
||||
}
|
||||
|
||||
void _mi_thread_locals_done(void) {
|
||||
mi_lock(&mi_thread_locals_lock) {
|
||||
mi_bitmap_t* const slots = mi_thread_locals_free;
|
||||
mi_free(slots);
|
||||
}
|
||||
mi_lock_done(&mi_thread_locals_lock);
|
||||
}
|
||||
|
||||
// strange signature but allows us to reuse the arena code for claiming free pages
|
||||
static bool mi_thread_local_claim_fun(size_t _slice_index, mi_arena_t* _arena, bool* keep_set) {
|
||||
MI_UNUSED(_slice_index); MI_UNUSED(_arena);
|
||||
*keep_set = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// When we claim a free slot, we increase the global version counter
|
||||
// (so if we reuse a slot it will be returning NULL initially when a thread tries to get it)
|
||||
static mi_thread_local_t mi_thread_local_claim(void) {
|
||||
size_t idx = 0;
|
||||
if (mi_thread_locals_free != NULL && mi_bitmap_try_find_and_claim(mi_thread_locals_free,0,&idx,&mi_thread_local_claim_fun,NULL)) {
|
||||
mi_thread_locals_version++;
|
||||
if (mi_thread_locals_version >= MI_TLS_VERSION_MAX) { mi_thread_locals_version = 1; } /* wrap around the version */
|
||||
return mi_key_create( idx, mi_thread_locals_version);
|
||||
}
|
||||
else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
static bool mi_thread_local_create_expand(void) {
|
||||
mi_bitmap_t* const slots = mi_thread_locals_free;
|
||||
// 1024 bits at a time
|
||||
const size_t oldcount = (slots==NULL ? 0 : mi_bitmap_max_bits(slots));
|
||||
const size_t newcount = 1024 + oldcount;
|
||||
if (newcount > MI_TLS_IDX_MAX) { return false; }
|
||||
const size_t newsize = mi_bitmap_size( newcount, NULL );
|
||||
mi_bitmap_t* newslots = (mi_bitmap_t*)mi_zalloc_aligned(newsize, MI_BCHUNK_SIZE);
|
||||
if (newslots==NULL) { return false; }
|
||||
if (slots!=NULL) {
|
||||
// copy over the previous bitmap
|
||||
_mi_memcpy_aligned(newslots, slots, mi_bitmap_size(oldcount, NULL));
|
||||
mi_free(slots);
|
||||
}
|
||||
mi_bitmap_init(newslots, newcount, true /* pretend already zero'd so we do not zero out the copied old entries */);
|
||||
mi_bitmap_unsafe_setN(newslots, oldcount, newcount - oldcount); /* set the new expanded slots as available */
|
||||
mi_thread_locals_free = newslots;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
// create a fresh key
|
||||
mi_thread_local_t _mi_thread_local_create(void) {
|
||||
mi_thread_local_t key = 0;
|
||||
mi_lock(&mi_thread_locals_lock) {
|
||||
key = mi_thread_local_claim();
|
||||
if (key==0) {
|
||||
if (mi_thread_local_create_expand()) {
|
||||
key = mi_thread_local_claim();
|
||||
}
|
||||
}
|
||||
}
|
||||
return key;
|
||||
}
|
||||
|
||||
// free a key
|
||||
void _mi_thread_local_free(mi_thread_local_t key) {
|
||||
if (key==0) return;
|
||||
const size_t idx = mi_key_index(key);
|
||||
mi_lock(&mi_thread_locals_lock) {
|
||||
mi_bitmap_t* const slots = mi_thread_locals_free;
|
||||
if (slots!=NULL && idx < mi_bitmap_max_bits(slots)) {
|
||||
mi_bitmap_set(slots,idx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user