chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,66 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2020 Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#pragma once
|
||||
#ifndef MIMALLOC_NEW_DELETE_H
|
||||
#define MIMALLOC_NEW_DELETE_H
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// This header provides convenient overrides for the new and
|
||||
// delete operations in C++.
|
||||
//
|
||||
// This header should be included in only one source file!
|
||||
//
|
||||
// On Windows, or when linking dynamically with mimalloc, these
|
||||
// can be more performant than the standard new-delete operations.
|
||||
// See <https://en.cppreference.com/w/cpp/memory/new/operator_new>
|
||||
// ---------------------------------------------------------------------------
|
||||
#if defined(__cplusplus)
|
||||
#include <new>
|
||||
#include <mimalloc.h>
|
||||
|
||||
#if defined(_MSC_VER) && defined(_Ret_notnull_) && defined(_Post_writable_byte_size_)
|
||||
// stay consistent with VCRT definitions
|
||||
#define mi_decl_new(n) mi_decl_nodiscard mi_decl_restrict _Ret_notnull_ _Post_writable_byte_size_(n)
|
||||
#define mi_decl_new_nothrow(n) mi_decl_nodiscard mi_decl_restrict _Ret_maybenull_ _Success_(return != NULL) _Post_writable_byte_size_(n)
|
||||
#else
|
||||
#define mi_decl_new(n) mi_decl_nodiscard mi_decl_restrict
|
||||
#define mi_decl_new_nothrow(n) mi_decl_nodiscard mi_decl_restrict
|
||||
#endif
|
||||
|
||||
void operator delete(void* p) noexcept { mi_free(p); };
|
||||
void operator delete[](void* p) noexcept { mi_free(p); };
|
||||
|
||||
void operator delete (void* p, const std::nothrow_t&) noexcept { mi_free(p); }
|
||||
void operator delete[](void* p, const std::nothrow_t&) noexcept { mi_free(p); }
|
||||
|
||||
mi_decl_new(n) void* operator new(std::size_t n) noexcept(false) { return mi_new(n); }
|
||||
mi_decl_new(n) void* operator new[](std::size_t n) noexcept(false) { return mi_new(n); }
|
||||
|
||||
mi_decl_new_nothrow(n) void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_new_nothrow(n); }
|
||||
mi_decl_new_nothrow(n) void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_new_nothrow(n); }
|
||||
|
||||
#if (__cplusplus >= 201402L || _MSC_VER >= 1916)
|
||||
void operator delete (void* p, std::size_t n) noexcept { mi_free_size(p,n); };
|
||||
void operator delete[](void* p, std::size_t n) noexcept { mi_free_size(p,n); };
|
||||
#endif
|
||||
|
||||
#if (__cplusplus > 201402L || defined(__cpp_aligned_new))
|
||||
void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
|
||||
void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
|
||||
void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
|
||||
void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
|
||||
void operator delete (void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
|
||||
void operator delete[](void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
|
||||
|
||||
void* operator new (std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
|
||||
void* operator new[](std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
|
||||
void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast<size_t>(al)); }
|
||||
void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast<size_t>(al)); }
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#endif // MIMALLOC_NEW_DELETE_H
|
||||
@@ -0,0 +1,68 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2020 Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#pragma once
|
||||
#ifndef MIMALLOC_OVERRIDE_H
|
||||
#define MIMALLOC_OVERRIDE_H
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
This header can be used to statically redirect malloc/free and new/delete
|
||||
to the mimalloc variants. This can be useful if one can include this file on
|
||||
each source file in a project (but be careful when using external code to
|
||||
not accidentally mix pointers from different allocators).
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
#include <mimalloc.h>
|
||||
|
||||
// Standard C allocation
|
||||
#define malloc(n) mi_malloc(n)
|
||||
#define calloc(c,n) mi_calloc(c,n)
|
||||
#define realloc(p,n) mi_realloc(p,n)
|
||||
#define free(p) mi_free(p)
|
||||
|
||||
#define strdup(s) mi_strdup(s)
|
||||
#define strndup(s,n) mi_strndup(s,n)
|
||||
#define realpath(f,n) mi_realpath(f,n)
|
||||
|
||||
// Microsoft extensions
|
||||
#define _expand(p,n) mi_expand(p,n)
|
||||
#define _msize(p) mi_usable_size(p)
|
||||
#define _recalloc(p,c,n) mi_recalloc(p,c,n)
|
||||
|
||||
#define _strdup(s) mi_strdup(s)
|
||||
#define _strndup(s,n) mi_strndup(s,n)
|
||||
#define _wcsdup(s) mi_wcsdup(s)
|
||||
#define _mbsdup(s) mi_mbsdup(s)
|
||||
#define _dupenv_s(buf,n,nm) mi_dupenv_s(buf,n,nm)
|
||||
#define _wdupenv_s(buf,n,nm) mi_wdupenv_s(buf,n,nm)
|
||||
|
||||
// Various Posix and Unix variants
|
||||
#define reallocf(p,n) mi_reallocf(p,n)
|
||||
#define malloc_size(p) mi_usable_size(p)
|
||||
#define malloc_usable_size(p) mi_usable_size(p)
|
||||
#define malloc_good_size(n) mi_malloc_good_size(n)
|
||||
#define cfree(p) mi_free(p)
|
||||
|
||||
#define valloc(n) mi_valloc(n)
|
||||
#define pvalloc(n) mi_pvalloc(n)
|
||||
#define reallocarray(p,c,n) mi_reallocarray(p,c,n)
|
||||
#define reallocarr(ptrp,c,n) mi_reallocarr(ptrp,c,n)
|
||||
#define memalign(a,n) mi_memalign(a,n)
|
||||
#define aligned_alloc(a,n) mi_aligned_alloc(a,n)
|
||||
#define posix_memalign(p,a,n) mi_posix_memalign(p,a,n)
|
||||
#define _posix_memalign(p,a,n) mi_posix_memalign(p,a,n)
|
||||
|
||||
// Microsoft aligned variants
|
||||
#define _aligned_malloc(n,a) mi_malloc_aligned(n,a)
|
||||
#define _aligned_realloc(p,n,a) mi_realloc_aligned(p,n,a)
|
||||
#define _aligned_recalloc(p,c,n,a) mi_aligned_recalloc(p,c,n,a)
|
||||
#define _aligned_msize(p,a,o) mi_usable_size(p)
|
||||
#define _aligned_free(p) mi_free(p)
|
||||
#define _aligned_offset_malloc(n,a,o) mi_malloc_aligned_at(n,a,o)
|
||||
#define _aligned_offset_realloc(p,n,a,o) mi_realloc_aligned_at(p,n,a,o)
|
||||
#define _aligned_offset_recalloc(p,c,n,a,o) mi_recalloc_aligned_at(p,c,n,a,o)
|
||||
|
||||
#endif // MIMALLOC_OVERRIDE_H
|
||||
@@ -0,0 +1,168 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2024-2025, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#pragma once
|
||||
#ifndef MIMALLOC_STATS_H
|
||||
#define MIMALLOC_STATS_H
|
||||
|
||||
#include <mimalloc.h>
|
||||
#include <string.h> // memset
|
||||
#include <stdint.h> // int64_t
|
||||
|
||||
#define MI_STAT_VERSION 5 // increased on every backward incompatible change
|
||||
|
||||
// alignment for atomic fields
|
||||
#if defined(_MSC_VER)
|
||||
#define mi_decl_align(a) __declspec(align(a))
|
||||
#elif defined(__GNUC__)
|
||||
#define mi_decl_align(a) __attribute__((aligned(a)))
|
||||
#elif __cplusplus >= 201103L
|
||||
#define mi_decl_align(a) alignas(a)
|
||||
#else
|
||||
#define mi_decl_align(a)
|
||||
#endif
|
||||
|
||||
|
||||
// count allocation over time
|
||||
typedef struct mi_stat_count_s {
|
||||
int64_t total; // total allocated
|
||||
int64_t peak; // peak allocation
|
||||
int64_t current; // current allocation
|
||||
} mi_stat_count_t;
|
||||
|
||||
// counters only increase
|
||||
typedef struct mi_stat_counter_s {
|
||||
int64_t total; // total count
|
||||
} mi_stat_counter_t;
|
||||
|
||||
#define MI_STAT_FIELDS() \
|
||||
MI_STAT_COUNT(pages) /* count of mimalloc pages */ \
|
||||
MI_STAT_COUNT(reserved) /* reserved memory bytes */ \
|
||||
MI_STAT_COUNT(committed) /* committed bytes */ \
|
||||
MI_STAT_COUNTER(reset) /* reset bytes */ \
|
||||
MI_STAT_COUNTER(purged) /* purged bytes */ \
|
||||
MI_STAT_COUNT(page_committed) /* committed memory inside pages */ \
|
||||
MI_STAT_COUNT(pages_abandoned) /* abandoned pages count */ \
|
||||
MI_STAT_COUNT(threads) /* number of threads */ \
|
||||
MI_STAT_COUNT(malloc_normal) /* allocated bytes <= MI_LARGE_OBJ_SIZE_MAX */ \
|
||||
MI_STAT_COUNT(malloc_huge) /* allocated bytes in huge pages */ \
|
||||
MI_STAT_COUNT(malloc_requested) /* malloc requested bytes */ \
|
||||
\
|
||||
MI_STAT_COUNTER(mmap_calls) \
|
||||
MI_STAT_COUNTER(commit_calls) \
|
||||
MI_STAT_COUNTER(reset_calls) \
|
||||
MI_STAT_COUNTER(purge_calls) \
|
||||
MI_STAT_COUNTER(arena_count) /* number of memory arena's */ \
|
||||
MI_STAT_COUNTER(malloc_normal_count) /* number of blocks <= MI_LARGE_OBJ_SIZE_MAX */ \
|
||||
MI_STAT_COUNTER(malloc_huge_count) /* number of huge bloks */ \
|
||||
MI_STAT_COUNTER(malloc_guarded_count) /* number of allocations with guard pages */ \
|
||||
\
|
||||
/* internal statistics */ \
|
||||
MI_STAT_COUNTER(arena_rollback_count) \
|
||||
MI_STAT_COUNTER(arena_purges) \
|
||||
MI_STAT_COUNTER(pages_extended) /* number of page extensions */ \
|
||||
MI_STAT_COUNTER(pages_retire) /* number of pages that are retired */ \
|
||||
MI_STAT_COUNTER(page_searches) /* total pages searched for a fresh page */ \
|
||||
MI_STAT_COUNTER(page_searches_count) /* searched count for a fresh page */ \
|
||||
/* only on v1 and v2 */ \
|
||||
MI_STAT_COUNT(segments) \
|
||||
MI_STAT_COUNT(segments_abandoned) \
|
||||
MI_STAT_COUNT(segments_cache) \
|
||||
MI_STAT_COUNT(_segments_reserved) \
|
||||
/* only on v3 */ \
|
||||
MI_STAT_COUNT(heaps) \
|
||||
MI_STAT_COUNT(theaps) \
|
||||
MI_STAT_COUNTER(pages_reclaim_on_alloc) \
|
||||
MI_STAT_COUNTER(pages_reclaim_on_free) \
|
||||
MI_STAT_COUNTER(pages_reabandon_full) \
|
||||
MI_STAT_COUNTER(pages_unabandon_busy_wait) \
|
||||
MI_STAT_COUNTER(heaps_delete_wait)
|
||||
|
||||
// Size bins for chunks
|
||||
typedef enum mi_chunkbin_e {
|
||||
MI_CBIN_SMALL, // slice_count == 1
|
||||
MI_CBIN_OTHER, // slice_count: any other from the other bins, and 1 <= slice_count <= MI_BCHUNK_BITS
|
||||
MI_CBIN_MEDIUM, // slice_count == 8
|
||||
MI_CBIN_LARGE, // slice_count == MI_SIZE_BITS (only used if MI_ENABLE_LARGE_PAGES is 1)
|
||||
MI_CBIN_HUGE, // slice_count > MI_BCHUNK_BITS
|
||||
MI_CBIN_NONE, // no bin assigned yet (the chunk is completely free)
|
||||
MI_CBIN_COUNT
|
||||
} mi_chunkbin_t;
|
||||
|
||||
|
||||
// Define the statistics structure
|
||||
#define MI_BIN_HUGE (73U) // see types.h
|
||||
#define MI_STAT_COUNT(stat) mi_stat_count_t stat;
|
||||
#define MI_STAT_COUNTER(stat) mi_stat_counter_t stat;
|
||||
|
||||
typedef struct mi_stats_s
|
||||
{
|
||||
size_t size; // size of the mi_stats_t structure
|
||||
size_t version;
|
||||
|
||||
mi_decl_align(8) MI_STAT_FIELDS()
|
||||
|
||||
// future extension
|
||||
mi_stat_count_t _stat_reserved[4];
|
||||
mi_stat_counter_t _stat_counter_reserved[4];
|
||||
|
||||
// size segregated statistics
|
||||
mi_stat_count_t malloc_bins[MI_BIN_HUGE+1]; // allocation per size bin
|
||||
mi_stat_count_t page_bins[MI_BIN_HUGE+1]; // pages allocated per size bin
|
||||
mi_stat_count_t chunk_bins[MI_CBIN_COUNT]; // chunks per page sizes
|
||||
} mi_stats_t;
|
||||
|
||||
#undef MI_STAT_COUNT
|
||||
#undef MI_STAT_COUNTER
|
||||
|
||||
// Initialization
|
||||
static inline void mi_stats_header_init(mi_stats_t* stats) {
|
||||
stats->size = sizeof(*stats);
|
||||
stats->version = MI_STAT_VERSION;
|
||||
}
|
||||
static inline void mi_stats_init(mi_stats_t* stats) {
|
||||
memset(stats,0,sizeof(*stats));
|
||||
mi_stats_header_init(stats);
|
||||
}
|
||||
|
||||
#define mi_stats_t_decl(name) mi_stats_t name; mi_stats_init(&name);
|
||||
|
||||
// Exported definitions
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
// stats from a heap
|
||||
mi_decl_export bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats) mi_attr_noexcept;
|
||||
mi_decl_export char* mi_heap_stats_get_json(mi_heap_t* heap, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL
|
||||
mi_decl_export void mi_heap_stats_print_out(mi_heap_t* heap, mi_output_fun* out, void* arg) mi_attr_noexcept;
|
||||
|
||||
// stats from a subprocess and its heaps aggregated
|
||||
mi_decl_export bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept;
|
||||
mi_decl_export char* mi_subproc_stats_get_json(mi_subproc_id_t subproc_id, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL
|
||||
mi_decl_export void mi_subproc_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg) mi_attr_noexcept;
|
||||
// print subprocess and all its heap stats segregated
|
||||
mi_decl_export void mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg) mi_attr_noexcept;
|
||||
|
||||
// stats aggregated for the current subprocess and all its heaps.
|
||||
mi_decl_export bool mi_stats_get(mi_stats_t* stats) mi_attr_noexcept;
|
||||
mi_decl_export char* mi_stats_get_json(size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL
|
||||
mi_decl_export void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
|
||||
|
||||
// add the stats of the heap to the subprocess and clear the heap stats
|
||||
mi_decl_export void mi_heap_stats_merge_to_subproc(mi_heap_t* heap);
|
||||
|
||||
// stats from the subprocess without aggregating its heaps
|
||||
mi_decl_export bool mi_subproc_stats_get_exclusive(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept;
|
||||
|
||||
mi_decl_export char* mi_stats_as_json(mi_stats_t* stats, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL
|
||||
mi_decl_export size_t mi_stats_get_bin_size(size_t bin) mi_attr_noexcept;
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // MIMALLOC_STATS_H
|
||||
@@ -0,0 +1,706 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#pragma once
|
||||
#ifndef MIMALLOC_H
|
||||
#define MIMALLOC_H
|
||||
|
||||
#define MI_MALLOC_VERSION 30302 // major + 2 digits minor + 2 digits patch
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Compiler specific attributes
|
||||
// ------------------------------------------------------
|
||||
|
||||
#ifdef __cplusplus
|
||||
#if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11
|
||||
#define mi_attr_noexcept noexcept
|
||||
#else
|
||||
#define mi_attr_noexcept throw()
|
||||
#endif
|
||||
#else
|
||||
#define mi_attr_noexcept
|
||||
#endif
|
||||
|
||||
#if defined(__cplusplus) && (__cplusplus >= 201703)
|
||||
#define mi_decl_nodiscard [[nodiscard]]
|
||||
#elif (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) // includes clang, icc, and clang-cl
|
||||
#define mi_decl_nodiscard __attribute__((warn_unused_result))
|
||||
#elif defined(_HAS_NODISCARD)
|
||||
#define mi_decl_nodiscard _NODISCARD
|
||||
#elif (_MSC_VER >= 1700)
|
||||
#define mi_decl_nodiscard _Check_return_
|
||||
#else
|
||||
#define mi_decl_nodiscard
|
||||
#endif
|
||||
|
||||
#if defined(_MSC_VER) || defined(__MINGW32__)
|
||||
#if !defined(MI_SHARED_LIB)
|
||||
#define mi_decl_export
|
||||
#elif defined(MI_SHARED_LIB_EXPORT)
|
||||
#define mi_decl_export __declspec(dllexport)
|
||||
#else
|
||||
#define mi_decl_export __declspec(dllimport)
|
||||
#endif
|
||||
#if defined(__MINGW32__)
|
||||
#define mi_decl_restrict
|
||||
#define mi_attr_malloc __attribute__((malloc))
|
||||
#else
|
||||
#if (_MSC_VER >= 1900) && !defined(__EDG__)
|
||||
#define mi_decl_restrict __declspec(allocator) __declspec(restrict)
|
||||
#else
|
||||
#define mi_decl_restrict __declspec(restrict)
|
||||
#endif
|
||||
#define mi_attr_malloc
|
||||
#endif
|
||||
#define mi_cdecl __cdecl
|
||||
#define mi_attr_alloc_size(s)
|
||||
#define mi_attr_alloc_size2(s1,s2)
|
||||
#define mi_attr_alloc_align(p)
|
||||
#elif defined(__GNUC__) // includes clang and icc
|
||||
#if defined(MI_SHARED_LIB) && defined(MI_SHARED_LIB_EXPORT)
|
||||
#define mi_decl_export __attribute__((visibility("default")))
|
||||
#else
|
||||
#define mi_decl_export
|
||||
#endif
|
||||
#define mi_cdecl // leads to warnings... __attribute__((cdecl))
|
||||
#define mi_decl_restrict
|
||||
#define mi_attr_malloc __attribute__((malloc))
|
||||
#if (defined(__clang_major__) && (__clang_major__ < 4)) || (__GNUC__ < 5)
|
||||
#define mi_attr_alloc_size(s)
|
||||
#define mi_attr_alloc_size2(s1,s2)
|
||||
#define mi_attr_alloc_align(p)
|
||||
#elif defined(__INTEL_COMPILER)
|
||||
#define mi_attr_alloc_size(s) __attribute__((alloc_size(s)))
|
||||
#define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2)))
|
||||
#define mi_attr_alloc_align(p)
|
||||
#else
|
||||
#define mi_attr_alloc_size(s) __attribute__((alloc_size(s)))
|
||||
#define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2)))
|
||||
#define mi_attr_alloc_align(p) __attribute__((alloc_align(p)))
|
||||
#endif
|
||||
#else
|
||||
#define mi_cdecl
|
||||
#define mi_decl_export
|
||||
#define mi_decl_restrict
|
||||
#define mi_attr_malloc
|
||||
#define mi_attr_alloc_size(s)
|
||||
#define mi_attr_alloc_size2(s1,s2)
|
||||
#define mi_attr_alloc_align(p)
|
||||
#endif
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Includes
|
||||
// ------------------------------------------------------
|
||||
|
||||
#include <stddef.h> // size_t
|
||||
#include <stdbool.h> // bool
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Standard malloc interface
|
||||
// ------------------------------------------------------
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);
|
||||
mi_decl_export void* mi_expand(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);
|
||||
|
||||
mi_decl_export void mi_free(void* p) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept mi_attr_malloc;
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept mi_attr_malloc;
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept;
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Extended allocation functions
|
||||
// ------------------------------------------------------
|
||||
#define MI_SMALL_WSIZE_MAX (128)
|
||||
#define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*))
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept;
|
||||
|
||||
// `mi_free_small` is for special applications like language runtimes.
|
||||
// it should only be used to free objects from `mi_(heap_)(m|z)alloc_small` and is potentially a tiny bit faster than `mi_free`
|
||||
mi_decl_export void mi_free_small(void* p) mi_attr_noexcept;
|
||||
|
||||
// -------------------------------------------------------------------------------------
|
||||
// Aligned allocation
|
||||
// Note that `alignment` always follows `size` for consistency with unaligned
|
||||
// allocation, but unfortunately this differs from `posix_memalign` and `aligned_alloc`.
|
||||
// -------------------------------------------------------------------------------------
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1, 2) mi_attr_alloc_align(3);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1, 2);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2);
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Typed allocation, the type is always the first parameter
|
||||
// ------------------------------------------------------
|
||||
|
||||
#define mi_malloc_tp(tp) ((tp*)mi_malloc(sizeof(tp)))
|
||||
#define mi_zalloc_tp(tp) ((tp*)mi_zalloc(sizeof(tp)))
|
||||
#define mi_calloc_tp(tp,n) ((tp*)mi_calloc(n,sizeof(tp)))
|
||||
#define mi_mallocn_tp(tp,n) ((tp*)mi_mallocn(n,sizeof(tp)))
|
||||
#define mi_reallocn_tp(tp,p,n) ((tp*)mi_reallocn(p,n,sizeof(tp)))
|
||||
#define mi_recalloc_tp(tp,p,n) ((tp*)mi_recalloc(p,n,sizeof(tp)))
|
||||
|
||||
#define mi_heap_malloc_tp(tp,hp) ((tp*)mi_heap_malloc(hp,sizeof(tp)))
|
||||
#define mi_heap_zalloc_tp(tp,hp) ((tp*)mi_heap_zalloc(hp,sizeof(tp)))
|
||||
#define mi_heap_calloc_tp(tp,hp,n) ((tp*)mi_heap_calloc(hp,n,sizeof(tp)))
|
||||
#define mi_heap_mallocn_tp(tp,hp,n) ((tp*)mi_heap_mallocn(hp,n,sizeof(tp)))
|
||||
#define mi_heap_reallocn_tp(tp,hp,p,n) ((tp*)mi_heap_reallocn(hp,p,n,sizeof(tp)))
|
||||
#define mi_heap_recalloc_tp(tp,hp,p,n) ((tp*)mi_heap_recalloc(hp,p,n,sizeof(tp)))
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Internals
|
||||
// See also `mimalloc-stats.h` for statistics
|
||||
// ------------------------------------------------------
|
||||
|
||||
typedef void (mi_cdecl mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg);
|
||||
mi_decl_export void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg) mi_attr_noexcept;
|
||||
|
||||
typedef void (mi_cdecl mi_output_fun)(const char* msg, void* arg);
|
||||
mi_decl_export void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept;
|
||||
|
||||
typedef void (mi_cdecl mi_error_fun)(int err, void* arg);
|
||||
mi_decl_export void mi_register_error(mi_error_fun* fun, void* arg);
|
||||
|
||||
mi_decl_export void mi_collect(bool force) mi_attr_noexcept;
|
||||
mi_decl_export int mi_version(void) mi_attr_noexcept;
|
||||
mi_decl_export void mi_options_print(void) mi_attr_noexcept;
|
||||
mi_decl_export void mi_process_info_print(void) mi_attr_noexcept;
|
||||
mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
|
||||
mi_decl_export void mi_process_info_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
|
||||
mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs,
|
||||
size_t* current_rss, size_t* peak_rss,
|
||||
size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept;
|
||||
|
||||
|
||||
|
||||
// Generally do not use the following as these are usually called automatically
|
||||
mi_decl_export void mi_process_init(void) mi_attr_noexcept;
|
||||
mi_decl_export void mi_cdecl mi_process_done(void) mi_attr_noexcept;
|
||||
mi_decl_export void mi_thread_init(void) mi_attr_noexcept;
|
||||
mi_decl_export void mi_thread_done(void) mi_attr_noexcept;
|
||||
mi_decl_export void mi_thread_set_in_threadpool(void) mi_attr_noexcept; // communicate that a thread is in a threadpool
|
||||
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
// Return allocated block size (if the return value is not NULL)
|
||||
// -----------------------------------------------------------------
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc(size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_urealloc(void* p, size_t newsize, size_t* block_size_pre, size_t* block_size_post) mi_attr_noexcept mi_attr_alloc_size(2);
|
||||
mi_decl_export void mi_ufree(void* p, size_t* block_size) mi_attr_noexcept;
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_uzalloc_small(size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
|
||||
|
||||
// -------------------------------------------------------------------------------------
|
||||
// Heaps: first-class. Can allocate from any thread (and be free'd from any thread)
|
||||
// Heaps keep allocations in separate pages from each other (but share the arena's and free'd pages)
|
||||
// -------------------------------------------------------------------------------------
|
||||
|
||||
struct mi_heap_s;
|
||||
typedef struct mi_heap_s mi_heap_t;
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new(void);
|
||||
mi_decl_export void mi_heap_delete(mi_heap_t* heap); // move live blocks to the main heap
|
||||
mi_decl_export void mi_heap_destroy(mi_heap_t* heap); // free all live blocks
|
||||
mi_decl_export void mi_heap_set_numa_affinity(mi_heap_t* heap, int numa_node);
|
||||
mi_decl_export void mi_heap_collect(mi_heap_t* heap, bool force);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_main(void);
|
||||
mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_of(const void* p);
|
||||
mi_decl_nodiscard mi_decl_export bool mi_heap_contains(const mi_heap_t* heap, const void* p);
|
||||
mi_decl_nodiscard mi_decl_export bool mi_any_heap_contains(const void* p);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc(mi_heap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3, 4);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_heap_reallocf(mi_heap_t* theap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept mi_attr_malloc;
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept mi_attr_malloc;
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept;
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3);
|
||||
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// Zero initialized re-allocation.
|
||||
// Only valid on memory that was originally allocated with zero initialization too.
|
||||
// e.g. `mi_calloc`, `mi_zalloc`, `mi_zalloc_aligned` etc.
|
||||
// see <https://github.com/microsoft/mimalloc/issues/63#issuecomment-508272992>
|
||||
// --------------------------------------------------------------------------------
|
||||
|
||||
mi_decl_nodiscard mi_decl_export void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_recalloc(void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(2,3) mi_attr_alloc_align(4);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(2,3);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3, 4);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(3, 4) mi_attr_alloc_align(5);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(3, 4);
|
||||
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Visiting pages and individual blocks in a heap.
|
||||
// ------------------------------------------------------
|
||||
|
||||
// An area of heap space contains blocks of a single size.
|
||||
typedef struct mi_heap_area_s {
|
||||
void* blocks; // start of the area containing theap blocks
|
||||
size_t reserved; // bytes reserved for this area (virtual)
|
||||
size_t committed; // current available bytes for this area
|
||||
size_t used; // number of allocated blocks
|
||||
size_t block_size; // size in bytes of each block
|
||||
size_t full_block_size; // size in bytes of a full block including padding and metadata.
|
||||
void* reserved1; // internal
|
||||
} mi_heap_area_t;
|
||||
|
||||
typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg);
|
||||
|
||||
mi_decl_export bool mi_heap_visit_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
|
||||
mi_decl_export bool mi_heap_visit_abandoned_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Arena memory management
|
||||
// Arena's are larger memory area's provided by the OS or user
|
||||
// ------------------------------------------------------
|
||||
|
||||
mi_decl_nodiscard mi_decl_export bool mi_is_redirected(void) mi_attr_noexcept;
|
||||
|
||||
mi_decl_export int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept;
|
||||
mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept;
|
||||
|
||||
mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept;
|
||||
mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_pinned /* cannot decommit/reset? */, bool is_zero, int numa_node) mi_attr_noexcept;
|
||||
|
||||
mi_decl_export void mi_debug_show_arenas(void) mi_attr_noexcept;
|
||||
mi_decl_export void mi_arenas_print(void) mi_attr_noexcept;
|
||||
mi_decl_export size_t mi_arena_min_alignment(void);
|
||||
mi_decl_export size_t mi_arena_min_size(void);
|
||||
|
||||
typedef void* mi_arena_id_t;
|
||||
mi_decl_export void* mi_arena_area(mi_arena_id_t arena_id, size_t* size);
|
||||
mi_decl_export int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;
|
||||
mi_decl_export int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;
|
||||
mi_decl_export bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;
|
||||
mi_decl_export bool mi_arena_contains(mi_arena_id_t arena_id, const void* p);
|
||||
|
||||
// Create a heap that only allocates in the specified arena
|
||||
mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id);
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Subprocesses
|
||||
// Advanced: allow sub-processes whose memory arena's stay fully separated (and no reclamation between them).
|
||||
// Used for example for separate interpreters in one process.
|
||||
// ------------------------------------------------------
|
||||
|
||||
typedef struct { void* _mi_subproc_id; } mi_subproc_id_t; // abstract type
|
||||
mi_decl_export mi_subproc_id_t mi_subproc_main(void);
|
||||
mi_decl_export mi_subproc_id_t mi_subproc_current(void);
|
||||
mi_decl_export mi_subproc_id_t mi_subproc_new(void);
|
||||
mi_decl_export void mi_subproc_destroy(mi_subproc_id_t subproc);
|
||||
mi_decl_export void mi_subproc_add_current_thread(mi_subproc_id_t subproc); // this should be called right after a thread is created (and no allocation has taken place yet)
|
||||
|
||||
typedef bool (mi_cdecl mi_heap_visit_fun)(mi_heap_t* heap, void* arg);
|
||||
mi_decl_export bool mi_subproc_visit_heaps(mi_subproc_id_t subproc, mi_heap_visit_fun* visitor, void* arg);
|
||||
|
||||
|
||||
// -------------------------------------------------------------------------------------
|
||||
// A "theap" is a thread-local heap. This API is only provided for special circumstances like runtimes
|
||||
// that already have a thread-local context and can store the theap there for (slightly) faster allocations.
|
||||
// This also allows to set a default theap for the current thread so that `malloc` etc. allocate from
|
||||
// that theap (instead of the main (t)heap).
|
||||
// Theaps are first-class, but can only allocate from the same thread that created it.
|
||||
// Allocation through a `theap` may be a tiny bit faster than using plain malloc
|
||||
// (as we don't need to lookup the thread local variable).
|
||||
// -------------------------------------------------------------------------------------
|
||||
|
||||
struct mi_theap_s;
|
||||
typedef struct mi_theap_s mi_theap_t;
|
||||
|
||||
mi_decl_export mi_theap_t* mi_heap_theap(mi_heap_t* heap);
|
||||
mi_decl_export mi_theap_t* mi_theap_set_default(mi_theap_t* theap);
|
||||
mi_decl_export mi_theap_t* mi_theap_get_default(void);
|
||||
mi_decl_export void mi_theap_collect(mi_theap_t* theap, bool force) mi_attr_noexcept;
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Experimental
|
||||
// ------------------------------------------------------
|
||||
|
||||
// Experimental: objects followed by a guard page.
|
||||
// Setting the sample rate on a specific theap can be used to test parts of the program more
|
||||
// specifically (in combination with `mi_theap_set_default`).
|
||||
// A sample rate of 0 disables guarded objects, while 1 uses a guard page for every object.
|
||||
// A seed of 0 uses a random start point. Only objects within the size bound are eligable for guard pages.
|
||||
mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed);
|
||||
mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max);
|
||||
|
||||
// very experimental
|
||||
typedef bool (mi_cdecl mi_commit_fun_t)(bool commit, void* start, size_t size, bool* is_zero, void* user_arg);
|
||||
mi_decl_export bool mi_manage_memory(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive,
|
||||
mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) mi_attr_noexcept;
|
||||
|
||||
//mi_decl_export bool mi_arena_unload(mi_arena_id_t arena_id, void** base, size_t* accessed_size, size_t* size);
|
||||
//mi_decl_export bool mi_arena_reload(void* start, size_t size, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id);
|
||||
//mi_decl_export bool mi_theap_reload(mi_theap_t* theap, mi_arena_id_t arena);
|
||||
//mi_decl_export void mi_theap_unload(mi_theap_t* theap);
|
||||
|
||||
// unsafe: assumes the page belonging to `p` is only accessed by the calling thread.
|
||||
mi_decl_export bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc_threshold) mi_attr_noexcept;
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Deprecated
|
||||
// ------------------------------------------------------
|
||||
|
||||
mi_decl_export bool mi_check_owned(const void* p);
|
||||
|
||||
mi_decl_export void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept;
|
||||
mi_decl_export bool mi_theap_visit_blocks(const mi_theap_t* theap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
|
||||
|
||||
mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept;
|
||||
mi_decl_export void mi_collect_reduce(size_t target_thread_owned) mi_attr_noexcept;
|
||||
|
||||
mi_decl_export void mi_stats_reset(void) mi_attr_noexcept;
|
||||
mi_decl_export void mi_stats_merge(void) mi_attr_noexcept;
|
||||
mi_decl_export void mi_stats_print(void* out) mi_attr_noexcept; // backward compatibility: `out` is ignored and should be NULL
|
||||
|
||||
mi_decl_export void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept; // not deprecated but declared in `mimalloc-stats.h` now.
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Options
|
||||
// ------------------------------------------------------
|
||||
|
||||
typedef enum mi_option_e {
|
||||
// stable options
|
||||
mi_option_show_errors, // print error messages
|
||||
mi_option_show_stats, // print statistics on termination
|
||||
mi_option_verbose, // print verbose messages
|
||||
// advanced options
|
||||
mi_option_deprecated_eager_commit,
|
||||
mi_option_arena_eager_commit, // eager commit arenas? Use 2 to enable just on overcommit systems (=2)
|
||||
mi_option_purge_decommits, // should a memory purge decommit? (=1). Set to 0 to use memory reset on a purge (instead of decommit)
|
||||
mi_option_allow_large_os_pages, // allow use of large (2 or 4 MiB) OS pages, implies eager commit.
|
||||
mi_option_reserve_huge_os_pages, // reserve N huge OS pages (1GiB pages) at startup
|
||||
mi_option_reserve_huge_os_pages_at, // reserve huge OS pages at a specific NUMA node
|
||||
mi_option_reserve_os_memory, // reserve specified amount of OS memory in an arena at startup (internally, this value is in KiB; use `mi_option_get_size`)
|
||||
mi_option_deprecated_segment_cache,
|
||||
mi_option_deprecated_page_reset,
|
||||
mi_option_deprecated_abandoned_page_purge,
|
||||
mi_option_deprecated_segment_reset,
|
||||
mi_option_deprecated_eager_commit_delay,
|
||||
mi_option_purge_delay, // memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all. (=10)
|
||||
mi_option_use_numa_nodes, // 0 = use all available numa nodes, otherwise use at most N nodes.
|
||||
mi_option_disallow_os_alloc, // 1 = do not use OS memory for allocation (but only programmatically reserved arenas)
|
||||
mi_option_os_tag, // tag used for OS logging (macOS only for now) (=100)
|
||||
mi_option_max_errors, // issue at most N error messages
|
||||
mi_option_max_warnings, // issue at most N warning messages
|
||||
mi_option_deprecated_max_segment_reclaim, // max. percentage of the abandoned segments can be reclaimed per try (=10%)
|
||||
mi_option_destroy_on_exit, // if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe
|
||||
mi_option_arena_reserve, // initial memory size for arena reservation (= 1 GiB on 64-bit) (internally, this value is in KiB; use `mi_option_get_size`)
|
||||
mi_option_arena_purge_mult, // multiplier for `purge_delay` for the purging delay for arenas (=10)
|
||||
mi_option_deprecated_purge_extend_delay,
|
||||
mi_option_disallow_arena_alloc, // 1 = do not use arena's for allocation (except if using specific arena id's)
|
||||
mi_option_retry_on_oom, // retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows)
|
||||
mi_option_visit_abandoned, // allow visiting theap blocks from abandoned threads (=0)
|
||||
mi_option_guarded_min, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects (=0)
|
||||
mi_option_guarded_max, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects (=0)
|
||||
mi_option_guarded_precise, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0)
|
||||
mi_option_guarded_sample_rate, // 1 out of N allocations in the min/max range will be guarded (=1000)
|
||||
mi_option_guarded_sample_seed, // can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0)
|
||||
mi_option_generic_collect, // collect theaps every N (=10000) generic allocation calls
|
||||
mi_option_page_reclaim_on_free, // reclaim abandoned pages on a free (=0). -1 disallowr always, 0 allows if the page originated from the current theap, 1 allow always
|
||||
mi_option_page_full_retain, // retain N full (small) pages per size class (=2)
|
||||
mi_option_page_max_candidates, // max candidate pages to consider for allocation (=4)
|
||||
mi_option_max_vabits, // max user space virtual address bits to consider (=48)
|
||||
mi_option_pagemap_commit, // commit the full pagemap (to always catch invalid pointer uses) (=0)
|
||||
mi_option_page_commit_on_demand, // commit page memory on-demand
|
||||
mi_option_page_max_reclaim, // don't reclaim pages of the same originating theap if we already own N pages (in that size class) (=-1 (unlimited))
|
||||
mi_option_page_cross_thread_max_reclaim, // don't reclaim pages across threads if we already own N pages (in that size class) (=16)
|
||||
mi_option_allow_thp, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process.
|
||||
mi_option_minimal_purge_size, // set minimal purge size (in KiB) (=0). By default set to either 64 or 2048 if THP is enabled.
|
||||
mi_option_arena_max_object_size, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit).
|
||||
mi_option_arena_is_numa_local, // experimental
|
||||
_mi_option_last,
|
||||
// legacy option names
|
||||
mi_option_large_os_pages = mi_option_allow_large_os_pages,
|
||||
mi_option_eager_region_commit = mi_option_arena_eager_commit,
|
||||
mi_option_reset_decommits = mi_option_purge_decommits,
|
||||
mi_option_reset_delay = mi_option_purge_delay,
|
||||
mi_option_limit_os_alloc = mi_option_disallow_os_alloc
|
||||
} mi_option_t;
|
||||
|
||||
|
||||
mi_decl_nodiscard mi_decl_export bool mi_option_is_enabled(mi_option_t option);
|
||||
mi_decl_export void mi_option_enable(mi_option_t option);
|
||||
mi_decl_export void mi_option_disable(mi_option_t option);
|
||||
mi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable);
|
||||
mi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option);
|
||||
mi_decl_nodiscard mi_decl_export long mi_option_get_clamp(mi_option_t option, long min, long max);
|
||||
mi_decl_nodiscard mi_decl_export size_t mi_option_get_size(mi_option_t option);
|
||||
mi_decl_export void mi_option_set(mi_option_t option, long value);
|
||||
mi_decl_export void mi_option_set_default(mi_option_t option, long value);
|
||||
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------
|
||||
// "mi" prefixed implementations of various posix, Unix, Windows, and C++ allocation functions.
|
||||
// (This can be convenient when providing overrides of these functions as done in `mimalloc-override.h`.)
|
||||
// note: we use `mi_cfree` as "checked free" and it checks if the pointer is in our theap before free-ing.
|
||||
// -------------------------------------------------------------------------------------------------------
|
||||
|
||||
mi_decl_export void mi_cfree(void* p) mi_attr_noexcept;
|
||||
mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept;
|
||||
|
||||
mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3);
|
||||
mi_decl_nodiscard mi_decl_export int mi_reallocarr(void* ptrp, size_t count, size_t size) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept;
|
||||
|
||||
mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept;
|
||||
mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept;
|
||||
mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept;
|
||||
mi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept;
|
||||
|
||||
// wide characters
|
||||
#include <wchar.h> // wchar_t
|
||||
mi_decl_export int mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s) mi_attr_noexcept mi_attr_malloc;
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept mi_attr_malloc;
|
||||
|
||||
// The `mi_new` wrappers implement C++ semantics on out-of-memory instead of directly returning `NULL`.
|
||||
// (and call `std::get_new_handler` and potentially raise a `std::bad_alloc` exception).
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new(size_t size) mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_n(size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(1, 2);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_new_realloc(void* p, size_t newsize) mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3);
|
||||
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Implement the C++ std::allocator interface for use in STL containers.
|
||||
// (note: see `mimalloc-new-delete.h` for overriding the new/delete operators globally)
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
#ifdef __cplusplus
|
||||
|
||||
#include <cstddef> // std::size_t
|
||||
#include <cstdint> // PTRDIFF_MAX
|
||||
#if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11
|
||||
#include <type_traits> // std::true_type
|
||||
#include <utility> // std::forward
|
||||
#endif
|
||||
|
||||
template<class T> struct _mi_stl_allocator_common {
|
||||
typedef T value_type;
|
||||
typedef std::size_t size_type;
|
||||
typedef std::ptrdiff_t difference_type;
|
||||
typedef value_type& reference;
|
||||
typedef value_type const& const_reference;
|
||||
typedef value_type* pointer;
|
||||
typedef value_type const* const_pointer;
|
||||
|
||||
#if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11
|
||||
using propagate_on_container_copy_assignment = std::true_type;
|
||||
using propagate_on_container_move_assignment = std::true_type;
|
||||
using propagate_on_container_swap = std::true_type;
|
||||
template <class U, class ...Args> void construct(U* p, Args&& ...args) { ::new(p) U(std::forward<Args>(args)...); }
|
||||
template <class U> void destroy(U* p) mi_attr_noexcept { p->~U(); }
|
||||
#else
|
||||
void construct(pointer p, value_type const& val) { ::new(p) value_type(val); }
|
||||
void destroy(pointer p) { p->~value_type(); }
|
||||
#endif
|
||||
|
||||
size_type max_size() const mi_attr_noexcept { return (PTRDIFF_MAX/sizeof(value_type)); }
|
||||
pointer address(reference x) const { return &x; }
|
||||
const_pointer address(const_reference x) const { return &x; }
|
||||
};
|
||||
|
||||
template<class T> struct mi_stl_allocator : public _mi_stl_allocator_common<T> {
|
||||
using typename _mi_stl_allocator_common<T>::size_type;
|
||||
using typename _mi_stl_allocator_common<T>::value_type;
|
||||
using typename _mi_stl_allocator_common<T>::pointer;
|
||||
template <class U> struct rebind { typedef mi_stl_allocator<U> other; };
|
||||
|
||||
mi_stl_allocator() mi_attr_noexcept = default;
|
||||
mi_stl_allocator(const mi_stl_allocator&) mi_attr_noexcept = default;
|
||||
template<class U> mi_stl_allocator(const mi_stl_allocator<U>&) mi_attr_noexcept { }
|
||||
mi_stl_allocator select_on_container_copy_construction() const { return *this; }
|
||||
void deallocate(T* p, size_type) { mi_free(p); }
|
||||
|
||||
#if (__cplusplus >= 201703L) // C++17
|
||||
mi_decl_nodiscard T* allocate(size_type count) { return static_cast<T*>(mi_new_n(count, sizeof(T))); }
|
||||
mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); }
|
||||
#else
|
||||
mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast<pointer>(mi_new_n(count, sizeof(value_type))); }
|
||||
#endif
|
||||
|
||||
#if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11
|
||||
using is_always_equal = std::true_type;
|
||||
#endif
|
||||
};
|
||||
|
||||
template<class T1,class T2> bool operator==(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return true; }
|
||||
template<class T1,class T2> bool operator!=(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return false; }
|
||||
|
||||
|
||||
#if (__cplusplus >= 201103L) || (_MSC_VER >= 1900) // C++11
|
||||
#define MI_HAS_HEAP_STL_ALLOCATOR 1
|
||||
|
||||
#include <memory> // std::shared_ptr
|
||||
|
||||
// Common base class for STL allocators in a specific theap
|
||||
template<class T, bool _mi_destroy> struct _mi_heap_stl_allocator_common : public _mi_stl_allocator_common<T> {
|
||||
using typename _mi_stl_allocator_common<T>::size_type;
|
||||
using typename _mi_stl_allocator_common<T>::value_type;
|
||||
using typename _mi_stl_allocator_common<T>::pointer;
|
||||
|
||||
_mi_heap_stl_allocator_common(mi_heap_t* hp) : heap(hp, [](mi_heap_t*) {}) {} /* will not delete nor destroy the passed in heap */
|
||||
|
||||
#if (__cplusplus >= 201703L) // C++17
|
||||
mi_decl_nodiscard T* allocate(size_type count) { return static_cast<T*>(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(T))); }
|
||||
mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); }
|
||||
#else
|
||||
mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast<pointer>(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(value_type))); }
|
||||
#endif
|
||||
|
||||
#if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11
|
||||
using is_always_equal = std::false_type;
|
||||
#endif
|
||||
|
||||
void collect(bool force) { mi_heap_collect(this->heap.get(), force); }
|
||||
template<class U> bool is_equal(const _mi_heap_stl_allocator_common<U, _mi_destroy>& x) const { return (this->heap == x.heap); }
|
||||
|
||||
protected:
|
||||
std::shared_ptr<mi_heap_t> heap;
|
||||
template<class U, bool D> friend struct _mi_heap_stl_allocator_common;
|
||||
|
||||
_mi_heap_stl_allocator_common() {
|
||||
mi_heap_t* hp = mi_heap_new();
|
||||
this->heap.reset(hp, (_mi_destroy ? &heap_destroy : &heap_delete)); /* calls heap_delete/destroy when the refcount drops to zero */
|
||||
}
|
||||
_mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common& x) mi_attr_noexcept : heap(x.heap) { }
|
||||
template<class U> _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common<U, _mi_destroy>& x) mi_attr_noexcept : heap(x.heap) { }
|
||||
|
||||
private:
|
||||
static void heap_delete(mi_heap_t* hp) { if (hp != NULL) { mi_heap_delete(hp); } }
|
||||
static void heap_destroy(mi_heap_t* hp) { if (hp != NULL) { mi_heap_destroy(hp); } }
|
||||
};
|
||||
|
||||
// STL allocator allocation in a specific heap
|
||||
template<class T> struct mi_heap_stl_allocator : public _mi_heap_stl_allocator_common<T, false> {
|
||||
using typename _mi_heap_stl_allocator_common<T, false>::size_type;
|
||||
mi_heap_stl_allocator() : _mi_heap_stl_allocator_common<T, false>() { } // creates fresh heap that is deleted when the destructor is called
|
||||
mi_heap_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common<T, false>(hp) { } // no delete nor destroy on the passed in heap
|
||||
template<class U> mi_heap_stl_allocator(const mi_heap_stl_allocator<U>& x) mi_attr_noexcept : _mi_heap_stl_allocator_common<T, false>(x) { }
|
||||
|
||||
mi_heap_stl_allocator select_on_container_copy_construction() const { return *this; }
|
||||
void deallocate(T* p, size_type) { mi_free(p); }
|
||||
template<class U> struct rebind { typedef mi_heap_stl_allocator<U> other; };
|
||||
};
|
||||
|
||||
template<class T1, class T2> bool operator==(const mi_heap_stl_allocator<T1>& x, const mi_heap_stl_allocator<T2>& y) mi_attr_noexcept { return (x.is_equal(y)); }
|
||||
template<class T1, class T2> bool operator!=(const mi_heap_stl_allocator<T1>& x, const mi_heap_stl_allocator<T2>& y) mi_attr_noexcept { return (!x.is_equal(y)); }
|
||||
|
||||
|
||||
// STL allocator allocation in a specific heap, where `free` does nothing and
|
||||
// the heap is destroyed in one go on destruction -- use with care!
|
||||
template<class T> struct mi_heap_destroy_stl_allocator : public _mi_heap_stl_allocator_common<T, true> {
|
||||
using typename _mi_heap_stl_allocator_common<T, true>::size_type;
|
||||
mi_heap_destroy_stl_allocator() : _mi_heap_stl_allocator_common<T, true>() { } // creates fresh heap that is destroyed when the destructor is called
|
||||
mi_heap_destroy_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common<T, true>(hp) { } // no delete nor destroy on the passed in heap
|
||||
template<class U> mi_heap_destroy_stl_allocator(const mi_heap_destroy_stl_allocator<U>& x) mi_attr_noexcept : _mi_heap_stl_allocator_common<T, true>(x) { }
|
||||
|
||||
mi_heap_destroy_stl_allocator select_on_container_copy_construction() const { return *this; }
|
||||
void deallocate(T*, size_type) { /* do nothing as we destroy the heap on destruct. */ }
|
||||
template<class U> struct rebind { typedef mi_heap_destroy_stl_allocator<U> other; };
|
||||
};
|
||||
|
||||
template<class T1, class T2> bool operator==(const mi_heap_destroy_stl_allocator<T1>& x, const mi_heap_destroy_stl_allocator<T2>& y) mi_attr_noexcept { return (x.is_equal(y)); }
|
||||
template<class T1, class T2> bool operator!=(const mi_heap_destroy_stl_allocator<T1>& x, const mi_heap_destroy_stl_allocator<T2>& y) mi_attr_noexcept { return (!x.is_equal(y)); }
|
||||
|
||||
#endif // C++11
|
||||
|
||||
#endif // __cplusplus
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,562 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2024 Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#pragma once
|
||||
#ifndef MI_ATOMIC_H
|
||||
#define MI_ATOMIC_H
|
||||
|
||||
// include windows.h or pthreads.h
|
||||
#if defined(_WIN32)
|
||||
#ifndef WIN32_LEAN_AND_MEAN
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#endif
|
||||
#include <windows.h>
|
||||
#elif !defined(__wasi__) && (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__))
|
||||
#define MI_USE_PTHREADS
|
||||
#include <pthread.h>
|
||||
#endif
|
||||
|
||||
// --------------------------------------------------------------------------------------------
|
||||
// Atomics
|
||||
// We need to be portable between C, C++, and MSVC.
|
||||
// We base the primitives on the C/C++ atomics and create a minimal wrapper for MSVC in C compilation mode.
|
||||
// This is why we try to use only `uintptr_t` and `<type>*` as atomic types.
|
||||
// To gain better insight in the range of used atomics, we use explicitly named memory order operations
|
||||
// instead of passing the memory order as a parameter.
|
||||
// -----------------------------------------------------------------------------------------------
|
||||
|
||||
#if defined(__cplusplus)
|
||||
// Use C++ atomics
|
||||
#include <atomic>
|
||||
#define _Atomic(tp) std::atomic<tp>
|
||||
#define mi_atomic(name) std::atomic_##name
|
||||
#define mi_memory_order(name) std::memory_order_##name
|
||||
#if (__cplusplus >= 202002L) // c++20, see issue #571
|
||||
#define MI_ATOMIC_VAR_INIT(x) x
|
||||
#elif !defined(ATOMIC_VAR_INIT)
|
||||
#define MI_ATOMIC_VAR_INIT(x) x
|
||||
#else
|
||||
#define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x)
|
||||
#endif
|
||||
#elif defined(_MSC_VER)
|
||||
// Use MSVC C wrapper for C11 atomics
|
||||
#define _Atomic(tp) tp
|
||||
#define MI_ATOMIC_VAR_INIT(x) x
|
||||
#define mi_atomic(name) mi_atomic_##name
|
||||
#define mi_memory_order(name) mi_memory_order_##name
|
||||
#else
|
||||
// Use C11 atomics
|
||||
#include <stdatomic.h>
|
||||
#define mi_atomic(name) atomic_##name
|
||||
#define mi_memory_order(name) memory_order_##name
|
||||
#if (__STDC_VERSION__ >= 201710L) // c17, see issue #735
|
||||
#define MI_ATOMIC_VAR_INIT(x) x
|
||||
#elif !defined(ATOMIC_VAR_INIT)
|
||||
#define MI_ATOMIC_VAR_INIT(x) x
|
||||
#else
|
||||
#define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x)
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// Various defines for all used memory orders in mimalloc
|
||||
#define mi_atomic_cas_weak(p,expected,desired,mem_success,mem_fail) \
|
||||
mi_atomic(compare_exchange_weak_explicit)(p,expected,desired,mem_success,mem_fail)
|
||||
|
||||
#define mi_atomic_cas_strong(p,expected,desired,mem_success,mem_fail) \
|
||||
mi_atomic(compare_exchange_strong_explicit)(p,expected,desired,mem_success,mem_fail)
|
||||
|
||||
#define mi_atomic_load_acquire(p) mi_atomic(load_explicit)(p,mi_memory_order(acquire))
|
||||
#define mi_atomic_load_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed))
|
||||
#define mi_atomic_store_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release))
|
||||
#define mi_atomic_store_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed))
|
||||
#define mi_atomic_exchange_relaxed(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(relaxed))
|
||||
#define mi_atomic_exchange_release(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(release))
|
||||
#define mi_atomic_exchange_acq_rel(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(acq_rel))
|
||||
|
||||
#define mi_atomic_cas_weak_relaxed(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(relaxed),mi_memory_order(relaxed))
|
||||
#define mi_atomic_cas_weak_release(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed))
|
||||
#define mi_atomic_cas_weak_acq_rel(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire))
|
||||
#define mi_atomic_cas_strong_relaxed(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(relaxed),mi_memory_order(relaxed))
|
||||
#define mi_atomic_cas_strong_release(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed))
|
||||
#define mi_atomic_cas_strong_acq_rel(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire))
|
||||
|
||||
#define mi_atomic_add_relaxed(p,x) mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(relaxed))
|
||||
#define mi_atomic_add_acq_rel(p,x) mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(acq_rel))
|
||||
#define mi_atomic_sub_relaxed(p,x) mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(relaxed))
|
||||
#define mi_atomic_sub_acq_rel(p,x) mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(acq_rel))
|
||||
#define mi_atomic_and_relaxed(p,x) mi_atomic(fetch_and_explicit)(p,x,mi_memory_order(relaxed))
|
||||
#define mi_atomic_and_acq_rel(p,x) mi_atomic(fetch_and_explicit)(p,x,mi_memory_order(acq_rel))
|
||||
#define mi_atomic_or_relaxed(p,x) mi_atomic(fetch_or_explicit)(p,x,mi_memory_order(relaxed))
|
||||
#define mi_atomic_or_acq_rel(p,x) mi_atomic(fetch_or_explicit)(p,x,mi_memory_order(acq_rel))
|
||||
|
||||
#define mi_atomic_increment_relaxed(p) mi_atomic_add_relaxed(p,(uintptr_t)1)
|
||||
#define mi_atomic_decrement_relaxed(p) mi_atomic_sub_relaxed(p,(uintptr_t)1)
|
||||
#define mi_atomic_increment_acq_rel(p) mi_atomic_add_acq_rel(p,(uintptr_t)1)
|
||||
#define mi_atomic_decrement_acq_rel(p) mi_atomic_sub_acq_rel(p,(uintptr_t)1)
|
||||
|
||||
static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add);
|
||||
static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub);
|
||||
|
||||
|
||||
#if defined(__cplusplus) || !defined(_MSC_VER)
|
||||
|
||||
// In C++/C11 atomics we have polymorphic atomics so can use the typed `ptr` variants (where `tp` is the type of atomic value)
|
||||
// We use these macros so we can provide a typed wrapper in MSVC in C compilation mode as well
|
||||
#define mi_atomic_load_ptr_acquire(tp,p) mi_atomic_load_acquire(p)
|
||||
#define mi_atomic_load_ptr_relaxed(tp,p) mi_atomic_load_relaxed(p)
|
||||
|
||||
// In C++ we need to add casts to help resolve templates if NULL is passed
|
||||
#if defined(__cplusplus)
|
||||
#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release(p,(tp*)x)
|
||||
#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed(p,(tp*)x)
|
||||
#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,(tp*)des)
|
||||
#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,(tp*)des)
|
||||
#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,(tp*)des)
|
||||
#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des) mi_atomic_cas_strong_acq_rel(p,exp,(tp*)des)
|
||||
#define mi_atomic_exchange_ptr_relaxed(tp,p,x) mi_atomic_exchange_relaxed(p,(tp*)x)
|
||||
#define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,(tp*)x)
|
||||
#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,(tp*)x)
|
||||
#else
|
||||
#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release(p,x)
|
||||
#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed(p,x)
|
||||
#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,des)
|
||||
#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,des)
|
||||
#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,des)
|
||||
#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des) mi_atomic_cas_strong_acq_rel(p,exp,des)
|
||||
#define mi_atomic_exchange_ptr_relaxed(tp,p,x) mi_atomic_exchange_relaxed(p,x)
|
||||
#define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,x)
|
||||
#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,x)
|
||||
#endif
|
||||
|
||||
// These are used by the statistics
|
||||
static inline int64_t mi_atomic_addi64_relaxed(volatile int64_t* p, int64_t add) {
|
||||
return mi_atomic(fetch_add_explicit)((_Atomic(int64_t)*)p, add, mi_memory_order(relaxed));
|
||||
}
|
||||
static inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const volatile int64_t* padd) {
|
||||
const int64_t add = mi_atomic_load_relaxed((_Atomic(int64_t)*)padd);
|
||||
if (add != 0) {
|
||||
mi_atomic(fetch_add_explicit)((_Atomic(int64_t)*)p, add, mi_memory_order(relaxed));
|
||||
}
|
||||
}
|
||||
static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) {
|
||||
int64_t current = mi_atomic_load_relaxed((_Atomic(int64_t)*)p);
|
||||
while (current < x && !mi_atomic_cas_weak_release((_Atomic(int64_t)*)p, ¤t, x)) { /* nothing */ };
|
||||
}
|
||||
|
||||
// Used by timers
|
||||
#define mi_atomic_loadi64_acquire(p) mi_atomic(load_explicit)(p,mi_memory_order(acquire))
|
||||
#define mi_atomic_loadi64_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed))
|
||||
#define mi_atomic_storei64_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release))
|
||||
#define mi_atomic_storei64_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed))
|
||||
|
||||
#define mi_atomic_casi64_strong_acq_rel(p,e,d) mi_atomic_cas_strong_acq_rel(p,e,d)
|
||||
#define mi_atomic_addi64_acq_rel(p,i) mi_atomic_add_acq_rel(p,i)
|
||||
|
||||
|
||||
#elif defined(_MSC_VER)
|
||||
|
||||
// Deprecated: MSVC plain C compilation wrapper that uses Interlocked operations to model C11 atomics.
|
||||
// It is recommended to always compile as C++ when using MSVC.
|
||||
|
||||
#include <intrin.h>
|
||||
#ifdef _WIN64
|
||||
typedef LONG64 msc_intptr_t;
|
||||
#define MI_MSC_64(f) f##64
|
||||
#define MI_MSC_XX(f) f##64
|
||||
#else
|
||||
typedef LONG msc_intptr_t;
|
||||
#define MI_MSC_64(f) f
|
||||
#define MI_MSC_XX(f) f##32
|
||||
#endif
|
||||
|
||||
typedef enum mi_memory_order_e {
|
||||
mi_memory_order_relaxed,
|
||||
mi_memory_order_consume,
|
||||
mi_memory_order_acquire,
|
||||
mi_memory_order_release,
|
||||
mi_memory_order_acq_rel,
|
||||
mi_memory_order_seq_cst
|
||||
} mi_memory_order;
|
||||
|
||||
static inline uintptr_t mi_atomic_fetch_add_explicit(_Atomic(uintptr_t)*p, uintptr_t add, mi_memory_order mo) {
|
||||
(void)(mo);
|
||||
return (uintptr_t)MI_MSC_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, (msc_intptr_t)add);
|
||||
}
|
||||
static inline uintptr_t mi_atomic_fetch_sub_explicit(_Atomic(uintptr_t)*p, uintptr_t sub, mi_memory_order mo) {
|
||||
(void)(mo);
|
||||
return (uintptr_t)MI_MSC_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, -((msc_intptr_t)sub));
|
||||
}
|
||||
static inline uintptr_t mi_atomic_fetch_and_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {
|
||||
(void)(mo);
|
||||
return (uintptr_t)MI_MSC_64(_InterlockedAnd)((volatile msc_intptr_t*)p, (msc_intptr_t)x);
|
||||
}
|
||||
static inline uintptr_t mi_atomic_fetch_or_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {
|
||||
(void)(mo);
|
||||
return (uintptr_t)MI_MSC_64(_InterlockedOr)((volatile msc_intptr_t*)p, (msc_intptr_t)x);
|
||||
}
|
||||
static inline bool mi_atomic_compare_exchange_strong_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) {
|
||||
(void)(mo1); (void)(mo2);
|
||||
const uintptr_t read = (uintptr_t)MI_MSC_64(_InterlockedCompareExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)desired, (msc_intptr_t)(*expected));
|
||||
if (read == *expected) {
|
||||
return true;
|
||||
}
|
||||
else {
|
||||
*expected = read;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
static inline bool mi_atomic_compare_exchange_weak_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) {
|
||||
return mi_atomic_compare_exchange_strong_explicit(p, expected, desired, mo1, mo2);
|
||||
}
|
||||
static inline uintptr_t mi_atomic_exchange_explicit(_Atomic(uintptr_t)*p, uintptr_t exchange, mi_memory_order mo) {
|
||||
(void)(mo);
|
||||
return (uintptr_t)MI_MSC_64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange);
|
||||
}
|
||||
static inline void mi_atomic_thread_fence(mi_memory_order mo) {
|
||||
(void)(mo);
|
||||
_Atomic(uintptr_t) x = 0;
|
||||
mi_atomic_exchange_explicit(&x, 1, mo);
|
||||
}
|
||||
|
||||
static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_memory_order mo) {
|
||||
(void)(mo);
|
||||
// assert(mo<=mi_memory_order_acquire); // others are not used by mimalloc
|
||||
#if defined(_M_IX86) || defined(_M_X64)
|
||||
return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
|
||||
#elif defined(_M_ARM) || defined(_M_ARM64)
|
||||
if (mo == mi_memory_order_relaxed) {
|
||||
return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
|
||||
}
|
||||
else if (mo <= mi_memory_order_acquire) {
|
||||
return MI_MSC_XX(__ldar)((volatile const uintptr_t*)p);
|
||||
}
|
||||
else {
|
||||
const uintptr_t u = (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
|
||||
__dmb(15); // _ARM(64)_BARRIER_SY
|
||||
return u;
|
||||
}
|
||||
#else
|
||||
#warning "define mi_atomic_load_explicit for MSVC C compilation on this platform (which should be readonly, see issue #1277)"
|
||||
return MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
|
||||
#endif
|
||||
}
|
||||
static inline void mi_atomic_store_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {
|
||||
(void)(mo);
|
||||
// assert(mo<=mi_memory_order_release); // others are not used by mimalloc
|
||||
#if defined(_M_IX86) || defined(_M_X64)
|
||||
MI_MSC_XX(__iso_volatile_store)((volatile intptr_t*)p, x);
|
||||
#elif defined(_M_ARM) || defined(_M_ARM64)
|
||||
if (mo == mi_memory_order_relaxed) {
|
||||
MI_MSC_XX(__iso_volatile_store)((volatile intptr_t*)p, x);
|
||||
}
|
||||
else if (mo <= mi_memory_order_release) {
|
||||
MI_MSC_XX(__stlr)((volatile uintptr_t*)p,x);
|
||||
}
|
||||
else {
|
||||
mi_atomic_exchange_explicit(p, x, mo);
|
||||
}
|
||||
#else
|
||||
mi_atomic_exchange_explicit(p, x, mo);
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline int64_t mi_atomic_loadi64_explicit(_Atomic(int64_t)*p, mi_memory_order mo) {
|
||||
(void)(mo);
|
||||
// assert(mo<=mi_memory_order_acquire); // others are not used by mimalloc
|
||||
#if defined(_M_IX86) || defined(_M_X64)
|
||||
return __iso_volatile_load64((volatile const int64_t*)p);
|
||||
#elif defined(_M_ARM) || defined(_M_ARM64)
|
||||
if (mo == mi_memory_order_relaxed) {
|
||||
return __iso_volatile_load64((volatile const int64_t*)p);
|
||||
}
|
||||
#if defined(_M_ARM64)
|
||||
else if (mo <= mi_memory_order_acquire) {
|
||||
return __ldar64((volatile const uintptr_t*)p);
|
||||
}
|
||||
#endif
|
||||
else {
|
||||
const int64_t i = __iso_volatile_load64((volatile const int64_t*)p);
|
||||
__dmb(15); // _ARM(64)_BARRIER_SY
|
||||
return i;
|
||||
}
|
||||
#else
|
||||
#warning "define mi_atomic_loadi64_explicit for MSVC C compilation on this platform (which should be readonly, see issue #1277)"
|
||||
return __iso_volatile_load64((volatile const int64_t*)p);
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline void mi_atomic_storei64_explicit(_Atomic(int64_t)*p, int64_t x, mi_memory_order mo) {
|
||||
(void)(mo);
|
||||
// assert(mo<=mi_memory_order_release); // others are not used by mimalloc
|
||||
#if defined(_M_IX86) || defined(_M_X64)
|
||||
__iso_volatile_store64((volatile int64_t*)p,x);
|
||||
#elif defined(_M_ARM) || defined(_M_ARM64)
|
||||
if (mo == mi_memory_order_relaxed) {
|
||||
__iso_volatile_store64((volatile int64_t*)p,x);
|
||||
}
|
||||
#if defined(_M_ARM64)
|
||||
else if (mo == mi_memory_order_release) {
|
||||
__stlr64((volatile uint64_t*)p, (uint64_t)x);
|
||||
}
|
||||
#endif
|
||||
else {
|
||||
InterlockedExchange64(p, x);
|
||||
}
|
||||
#else
|
||||
InterlockedExchange64(p, x);
|
||||
#endif
|
||||
}
|
||||
|
||||
// These are used by the statistics
|
||||
static inline int64_t mi_atomic_addi64_relaxed(volatile _Atomic(int64_t)*p, int64_t add) {
|
||||
#ifdef _WIN64
|
||||
return (int64_t)mi_atomic_addi((int64_t*)p, add);
|
||||
#elif defined(_M_ARM)
|
||||
return _InterlockedExchangeAdd64(p, add);
|
||||
#else
|
||||
// x86
|
||||
int64_t current;
|
||||
int64_t sum;
|
||||
do {
|
||||
current = __iso_volatile_load64((volatile const int64_t*)p);
|
||||
sum = current + add;
|
||||
} while (_InterlockedCompareExchange64(p, sum, current) != current);
|
||||
return current;
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const volatile int64_t* padd) {
|
||||
const int64_t add = *padd;
|
||||
if (add != 0) {
|
||||
mi_atomic_addi64_relaxed((volatile _Atomic(int64_t)*)p, add);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)*p, int64_t x) {
|
||||
int64_t current;
|
||||
do {
|
||||
current = *p;
|
||||
} while (current < x && _InterlockedCompareExchange64(p, x, current) != current);
|
||||
}
|
||||
|
||||
static inline void mi_atomic_addi64_acq_rel(volatile _Atomic(int64_t*)p, int64_t i) {
|
||||
mi_atomic_addi64_relaxed(p, i);
|
||||
}
|
||||
|
||||
static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t*)p, int64_t* exp, int64_t des) {
|
||||
const int64_t read = _InterlockedCompareExchange64(p, des, *exp);
|
||||
if (read == *exp) {
|
||||
return true;
|
||||
}
|
||||
else {
|
||||
*exp = read;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// The pointer macros cast to `uintptr_t`.
|
||||
#define mi_atomic_load_ptr_acquire(tp,p) (tp*)mi_atomic_load_acquire((_Atomic(uintptr_t)*)(p))
|
||||
#define mi_atomic_load_ptr_relaxed(tp,p) (tp*)mi_atomic_load_relaxed((_Atomic(uintptr_t)*)(p))
|
||||
#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release((_Atomic(uintptr_t)*)(p),(uintptr_t)(x))
|
||||
#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)(x))
|
||||
#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
|
||||
#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
|
||||
#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
|
||||
#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des) mi_atomic_cas_strong_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
|
||||
#define mi_atomic_exchange_ptr_relaxed(tp,p,x) (tp*)mi_atomic_exchange_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)x)
|
||||
#define mi_atomic_exchange_ptr_release(tp,p,x) (tp*)mi_atomic_exchange_release((_Atomic(uintptr_t)*)(p),(uintptr_t)x)
|
||||
#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) (tp*)mi_atomic_exchange_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t)x)
|
||||
|
||||
#define mi_atomic_loadi64_acquire(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(acquire))
|
||||
#define mi_atomic_loadi64_relaxed(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(relaxed))
|
||||
#define mi_atomic_storei64_release(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(release))
|
||||
#define mi_atomic_storei64_relaxed(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(relaxed))
|
||||
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
// Atomically add a signed value; returns the previous value.
|
||||
static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add) {
|
||||
return (intptr_t)mi_atomic_add_acq_rel((_Atomic(uintptr_t)*)p, (uintptr_t)add);
|
||||
}
|
||||
|
||||
// Atomically subtract a signed value; returns the previous value.
|
||||
static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub) {
|
||||
return (intptr_t)mi_atomic_addi(p, -sub);
|
||||
}
|
||||
|
||||
|
||||
// ----------------------------------------------------------------------
|
||||
// Guard
|
||||
// ----------------------------------------------------------------------
|
||||
|
||||
typedef _Atomic(uintptr_t) mi_atomic_guard_t;
|
||||
|
||||
// Allows only one thread to execute at a time (without blocking anyone)
|
||||
#define mi_atomic_guard(guard) \
|
||||
uintptr_t _mi_guard_expected = 0; \
|
||||
for(bool _mi_guard_once = true; \
|
||||
_mi_guard_once && mi_atomic_cas_strong_acq_rel(guard,&_mi_guard_expected,(uintptr_t)1); \
|
||||
(mi_atomic_store_release(guard,(uintptr_t)0), _mi_guard_once = false) )
|
||||
|
||||
|
||||
// ----------------------------------------------------------------------
|
||||
// Locks
|
||||
// These should be light-weight in-process only locks.
|
||||
// Only used for reserving arena's and to maintain the abandoned list.
|
||||
// ----------------------------------------------------------------------
|
||||
#if _MSC_VER
|
||||
#pragma warning(disable:26110) // unlock with holding lock
|
||||
#endif
|
||||
|
||||
#define mi_lock(lock) for(bool _mi_go = (mi_lock_acquire(lock),true); _mi_go; (mi_lock_release(lock), _mi_go=false) )
|
||||
#define mi_lock_maybe(lock,acquire) for(bool _mi_go = (acquire ? (mi_lock_acquire(lock),true) : true); _mi_go; _mi_go = (acquire ? (mi_lock_release(lock),false) : false) )
|
||||
|
||||
|
||||
#if defined(_WIN32)
|
||||
|
||||
typedef struct mi_lock_s {
|
||||
SRWLOCK mutex; // slim reader-writer lock
|
||||
} mi_lock_t;
|
||||
|
||||
#define MI_LOCK_INITIALIZER { SRWLOCK_INIT }
|
||||
|
||||
static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
|
||||
return TryAcquireSRWLockExclusive(&lock->mutex);
|
||||
}
|
||||
static inline void mi_lock_acquire(mi_lock_t* lock) {
|
||||
AcquireSRWLockExclusive(&lock->mutex);
|
||||
}
|
||||
static inline void mi_lock_release(mi_lock_t* lock) {
|
||||
ReleaseSRWLockExclusive(&lock->mutex);
|
||||
}
|
||||
static inline void mi_lock_init(mi_lock_t* lock) {
|
||||
InitializeSRWLock(&lock->mutex);
|
||||
}
|
||||
static inline void mi_lock_done(mi_lock_t* lock) {
|
||||
(void)(lock);
|
||||
}
|
||||
|
||||
#elif defined(MI_USE_PTHREADS)
|
||||
|
||||
#include <string.h> // memcpy
|
||||
void _mi_error_message(int err, const char* fmt, ...);
|
||||
|
||||
typedef struct mi_lock_s {
|
||||
pthread_mutex_t mutex;
|
||||
} mi_lock_t;
|
||||
|
||||
#define MI_LOCK_INITIALIZER { PTHREAD_MUTEX_INITIALIZER }
|
||||
|
||||
static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
|
||||
return (pthread_mutex_trylock(&lock->mutex) == 0);
|
||||
}
|
||||
static inline void mi_lock_acquire(mi_lock_t* lock) {
|
||||
const int err = pthread_mutex_lock(&lock->mutex);
|
||||
if (err != 0) {
|
||||
_mi_error_message(err, "internal error: lock cannot be acquired (err %i)\n", err);
|
||||
}
|
||||
}
|
||||
static inline void mi_lock_release(mi_lock_t* lock) {
|
||||
pthread_mutex_unlock(&lock->mutex);
|
||||
}
|
||||
static inline void mi_lock_init(mi_lock_t* lock) {
|
||||
if(lock==NULL) return;
|
||||
// use this instead of pthread_mutex_init since that can cause allocation on some platforms (and recursively initialize)
|
||||
const pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
memcpy(&lock->mutex,&mutex,sizeof(mutex));
|
||||
}
|
||||
static inline void mi_lock_done(mi_lock_t* lock) {
|
||||
pthread_mutex_destroy(&lock->mutex);
|
||||
}
|
||||
|
||||
#elif defined(__cplusplus)
|
||||
|
||||
#include <thread>
|
||||
#include <mutex>
|
||||
#include <new>
|
||||
|
||||
typedef struct mi_lock_s {
|
||||
std::mutex mutex;
|
||||
} mi_lock_t;
|
||||
|
||||
#define MI_LOCK_INITIALIZER { }
|
||||
|
||||
static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
|
||||
return lock->mutex.try_lock();
|
||||
}
|
||||
static inline void mi_lock_acquire(mi_lock_t* lock) {
|
||||
lock->mutex.lock();
|
||||
}
|
||||
static inline void mi_lock_release(mi_lock_t* lock) {
|
||||
lock->mutex.unlock();
|
||||
}
|
||||
static inline void mi_lock_init(mi_lock_t* lock) {
|
||||
new(&lock->mutex) std::mutex();
|
||||
}
|
||||
static inline void mi_lock_done(mi_lock_t* lock) {
|
||||
(void)(lock);
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
// fall back to poor man's locks.
|
||||
// this should only be the case in a single-threaded environment (like __wasi__)
|
||||
#include <errno.h>
|
||||
#ifndef EFAULT
|
||||
#define EFAULT (14)
|
||||
#endif
|
||||
void _mi_error_message(int err, const char* fmt, ...);
|
||||
void _mi_prim_thread_yield(void);
|
||||
|
||||
typedef struct mi_lock_s {
|
||||
_Atomic(uintptr_t) mutex;
|
||||
} mi_lock_t;
|
||||
|
||||
#define MI_LOCK_INITIALIZER { MI_ATOMIC_VAR_INIT(0) }
|
||||
|
||||
static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
|
||||
uintptr_t expected = 0;
|
||||
return mi_atomic_cas_strong_acq_rel(&lock->mutex, &expected, (uintptr_t)1);
|
||||
}
|
||||
static inline void mi_lock_acquire(mi_lock_t* lock) {
|
||||
for (int i = 0; i < 10000; i++) { // for at most 10000 tries?
|
||||
if (mi_lock_try_acquire(lock)) return;
|
||||
_mi_prim_thread_yield();
|
||||
}
|
||||
_mi_error_message(EFAULT, "internal error: lock cannot be acquired (due to lack of native lock primitives)\n");
|
||||
}
|
||||
static inline void mi_lock_release(mi_lock_t* lock) {
|
||||
mi_atomic_store_release(&lock->mutex, (uintptr_t)0);
|
||||
}
|
||||
static inline void mi_lock_init(mi_lock_t* lock) {
|
||||
mi_lock_release(lock);
|
||||
}
|
||||
static inline void mi_lock_done(mi_lock_t* lock) {
|
||||
(void)(lock);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
typedef struct mi_atomic_once_s {
|
||||
_Atomic(uintptr_t) tid;
|
||||
mi_lock_t lock;
|
||||
} mi_atomic_once_t;
|
||||
|
||||
// Returns `true` only on the first invocation, signifying we can execute an action once.
|
||||
// If it returns `true`, the caller should call `_mi_atomic_once_release` after performing the action.
|
||||
// Other threads (than the initial thread that entered) will block until `_mi_atomic_once_release` has been called.
|
||||
bool _mi_atomic_once_enter(mi_atomic_once_t* once); // defined in `libc.c`
|
||||
void _mi_atomic_once_release(mi_atomic_once_t* once); // defined in `libc.c`
|
||||
|
||||
#define mi_atomic_do_once \
|
||||
static mi_atomic_once_t _mi_once = { MI_ATOMIC_VAR_INIT(0), MI_LOCK_INITIALIZER }; \
|
||||
for(bool _mi_exec = _mi_atomic_once_enter(&_mi_once); _mi_exec; (_mi_atomic_once_release(&_mi_once),_mi_exec=false))
|
||||
|
||||
|
||||
#endif // __MIMALLOC_ATOMIC_H
|
||||
@@ -0,0 +1,340 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2019-2024 Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
Bit operation, and platform dependent definition (MI_INTPTR_SIZE etc)
|
||||
---------------------------------------------------------------------------- */
|
||||
|
||||
#pragma once
|
||||
#ifndef MI_BITS_H
|
||||
#define MI_BITS_H
|
||||
|
||||
#include <stddef.h> // size_t
|
||||
#include <stdint.h> // int64_t etc
|
||||
#include <stdbool.h> // bool
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Size of a pointer.
|
||||
// We assume that `sizeof(void*)==sizeof(intptr_t)`
|
||||
// and it holds for all platforms we know of.
|
||||
//
|
||||
// However, the C standard only requires that:
|
||||
// p == (void*)((intptr_t)p))
|
||||
// but we also need:
|
||||
// i == (intptr_t)((void*)i)
|
||||
// or otherwise one might define an intptr_t type that is larger than a pointer...
|
||||
// ------------------------------------------------------
|
||||
|
||||
#if INTPTR_MAX > INT64_MAX
|
||||
# define MI_INTPTR_SHIFT (4) // assume 128-bit (as on arm CHERI for example)
|
||||
#elif INTPTR_MAX == INT64_MAX
|
||||
# define MI_INTPTR_SHIFT (3)
|
||||
#elif INTPTR_MAX == INT32_MAX
|
||||
# define MI_INTPTR_SHIFT (2)
|
||||
#else
|
||||
#error platform pointers must be 32, 64, or 128 bits
|
||||
#endif
|
||||
|
||||
#if (INTPTR_MAX) > LONG_MAX
|
||||
# define MI_PU(x) x##ULL
|
||||
#else
|
||||
# define MI_PU(x) x##UL
|
||||
#endif
|
||||
|
||||
#if SIZE_MAX == UINT64_MAX
|
||||
# define MI_SIZE_SHIFT (3)
|
||||
typedef int64_t mi_ssize_t;
|
||||
#elif SIZE_MAX == UINT32_MAX
|
||||
# define MI_SIZE_SHIFT (2)
|
||||
typedef int32_t mi_ssize_t;
|
||||
#else
|
||||
#error platform objects must be 32 or 64 bits in size
|
||||
#endif
|
||||
|
||||
#if (SIZE_MAX/2) > LONG_MAX
|
||||
# define MI_ZU(x) x##ULL
|
||||
#else
|
||||
# define MI_ZU(x) x##UL
|
||||
#endif
|
||||
|
||||
#define MI_INTPTR_SIZE (1<<MI_INTPTR_SHIFT)
|
||||
#define MI_INTPTR_BITS (MI_INTPTR_SIZE*8)
|
||||
|
||||
#define MI_SIZE_SIZE (1<<MI_SIZE_SHIFT)
|
||||
#define MI_SIZE_BITS (MI_SIZE_SIZE*8)
|
||||
|
||||
#define MI_KiB (MI_ZU(1024))
|
||||
#define MI_MiB (MI_KiB*MI_KiB)
|
||||
#define MI_GiB (MI_MiB*MI_KiB)
|
||||
|
||||
|
||||
/* --------------------------------------------------------------------------------
|
||||
Architecture
|
||||
-------------------------------------------------------------------------------- */
|
||||
|
||||
#if defined(__aarch64__) || defined(_M_ARM64) || defined(_M_HYBRID_X86_ARM64) || defined(_M_ARM64EC) // consider arm64ec as arm64
|
||||
#define MI_ARCH_ARM64 1
|
||||
#elif defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64)
|
||||
#define MI_ARCH_X64 1
|
||||
#elif defined(__i386__) || defined(__i386) || defined(_M_IX86) || defined(_X86_) || defined(__X86__)
|
||||
#define MI_ARCH_X86 1
|
||||
#elif defined(__arm__) || defined(_ARM) || defined(_M_ARM) || defined(_M_ARMT) || defined(__arm)
|
||||
#define MI_ARCH_ARM32 1
|
||||
#elif defined(__riscv) || defined(_M_RISCV)
|
||||
#define MI_ARCH_RISCV 1
|
||||
#if (LONG_MAX == INT32_MAX)
|
||||
#define MI_ARCH_RISCV32 1
|
||||
#else
|
||||
#define MI_ARCH_RISCV64 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if MI_ARCH_X64 && defined(__AVX2__)
|
||||
#include <immintrin.h>
|
||||
#elif MI_ARCH_ARM64 && MI_OPT_SIMD
|
||||
#include <arm_neon.h>
|
||||
#endif
|
||||
#if defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
|
||||
#include <intrin.h>
|
||||
#endif
|
||||
|
||||
#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__BMI2__) // msvc
|
||||
#define __BMI2__ 1
|
||||
#endif
|
||||
#if MI_ARCH_X64 && (defined(__AVX2__) || defined(__BMI2__)) && !defined(__BMI1__) // msvc
|
||||
#define __BMI1__ 1
|
||||
#endif
|
||||
#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__LZCNT__) // msvc
|
||||
#define __LZCNT__ 1
|
||||
#endif
|
||||
|
||||
// Define big endian if needed
|
||||
// #define MI_BIG_ENDIAN 1
|
||||
|
||||
// maximum virtual address bits in a user-space pointer
|
||||
#if MI_DEFAULT_VIRTUAL_ADDRESS_BITS > 0
|
||||
#define MI_MAX_VABITS MI_DEFAULT_VIRTUAL_ADDRESS_BITS
|
||||
#elif MI_ARCH_X64
|
||||
#define MI_MAX_VABITS (47)
|
||||
#elif MI_INTPTR_SIZE > 4
|
||||
#define MI_MAX_VABITS (48)
|
||||
#else
|
||||
#define MI_MAX_VABITS (32)
|
||||
#endif
|
||||
|
||||
// use a flat page-map or a 2-level one
|
||||
#ifndef MI_PAGE_MAP_FLAT
|
||||
#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && MI_SECURE==0 && !MI_PAGE_META_IS_SEPARATED
|
||||
#define MI_PAGE_MAP_FLAT 1
|
||||
#else
|
||||
#define MI_PAGE_MAP_FLAT 0
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
||||
/* --------------------------------------------------------------------------------
|
||||
Builtin's
|
||||
-------------------------------------------------------------------------------- */
|
||||
|
||||
#ifndef __has_builtin
|
||||
#define __has_builtin(x) 0
|
||||
#endif
|
||||
|
||||
#define mi_builtin(name) __builtin_##name
|
||||
#define mi_has_builtin(name) __has_builtin(__builtin_##name)
|
||||
|
||||
#if (LONG_MAX == INT32_MAX)
|
||||
#define mi_builtin32(name) mi_builtin(name##l)
|
||||
#define mi_has_builtin32(name) mi_has_builtin(name##l)
|
||||
#else
|
||||
#define mi_builtin32(name) mi_builtin(name)
|
||||
#define mi_has_builtin32(name) mi_has_builtin(name)
|
||||
#endif
|
||||
#if (LONG_MAX == INT64_MAX)
|
||||
#define mi_builtin64(name) mi_builtin(name##l)
|
||||
#define mi_has_builtin64(name) mi_has_builtin(name##l)
|
||||
#else
|
||||
#define mi_builtin64(name) mi_builtin(name##ll)
|
||||
#define mi_has_builtin64(name) mi_has_builtin(name##ll)
|
||||
#endif
|
||||
|
||||
#if (MI_SIZE_BITS == 32)
|
||||
#define mi_builtinz(name) mi_builtin32(name)
|
||||
#define mi_has_builtinz(name) mi_has_builtin32(name)
|
||||
#define mi_msc_builtinz(name) name
|
||||
#elif (MI_SIZE_BITS == 64)
|
||||
#define mi_builtinz(name) mi_builtin64(name)
|
||||
#define mi_has_builtinz(name) mi_has_builtin64(name)
|
||||
#define mi_msc_builtinz(name) name##64
|
||||
#endif
|
||||
|
||||
/* --------------------------------------------------------------------------------
|
||||
Popcount and count trailing/leading zero's
|
||||
-------------------------------------------------------------------------------- */
|
||||
|
||||
size_t _mi_popcount_generic(size_t x);
|
||||
|
||||
static inline size_t mi_popcount(size_t x) {
|
||||
#if mi_has_builtinz(popcount)
|
||||
return mi_builtinz(popcount)(x);
|
||||
#elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
|
||||
return mi_msc_builtinz(__popcnt)(x);
|
||||
#elif MI_ARCH_X64 && defined(__BMI1__)
|
||||
return (size_t)_mm_popcnt_u64(x);
|
||||
#else
|
||||
#define MI_HAS_FAST_POPCOUNT 0
|
||||
return (x<=1 ? x : _mi_popcount_generic(x));
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifndef MI_HAS_FAST_POPCOUNT
|
||||
#define MI_HAS_FAST_POPCOUNT 1
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
size_t _mi_clz_generic(size_t x);
|
||||
size_t _mi_ctz_generic(size_t x);
|
||||
|
||||
static inline size_t mi_ctz(size_t x) {
|
||||
#if defined(__GNUC__) && MI_ARCH_X64 && defined(__BMI1__) // on x64 tzcnt is defined for 0
|
||||
size_t r;
|
||||
__asm ("tzcnt\t%1, %0" : "=r"(r) : "r"(x) : "cc");
|
||||
return r;
|
||||
#elif defined(_MSC_VER) && MI_ARCH_X64 && defined(__BMI1__)
|
||||
return _tzcnt_u64(x);
|
||||
#elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
|
||||
unsigned long idx;
|
||||
return (mi_msc_builtinz(_BitScanForward)(&idx, x) ? (size_t)idx : MI_SIZE_BITS);
|
||||
#elif mi_has_builtinz(ctz)
|
||||
return (x!=0 ? (size_t)mi_builtinz(ctz)(x) : MI_SIZE_BITS);
|
||||
#elif defined(__GNUC__) && (MI_ARCH_X64 || MI_ARCH_X86)
|
||||
size_t r = MI_SIZE_BITS; // bsf leaves destination unmodified if the argument is 0 (see <https://github.com/llvm/llvm-project/pull/102885>)
|
||||
__asm ("bsf\t%1, %0" : "+r"(r) : "r"(x) : "cc");
|
||||
return r;
|
||||
#elif MI_HAS_FAST_POPCOUNT
|
||||
return (x!=0 ? (mi_popcount(x^(x-1))-1) : MI_SIZE_BITS);
|
||||
#else
|
||||
#define MI_HAS_FAST_BITSCAN 0
|
||||
return (x!=0 ? _mi_ctz_generic(x) : MI_SIZE_BITS);
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline size_t mi_clz(size_t x) {
|
||||
#if defined(__GNUC__) && MI_ARCH_X64 && defined(__LZCNT__) // on x64 lzcnt is defined for 0
|
||||
size_t r;
|
||||
__asm ("lzcnt\t%1, %0" : "=r"(r) : "r"(x) : "cc");
|
||||
return r;
|
||||
#elif defined(_MSC_VER) && MI_ARCH_X64 && defined(__LZCNT__)
|
||||
return _lzcnt_u64(x);
|
||||
#elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
|
||||
unsigned long idx;
|
||||
return (mi_msc_builtinz(_BitScanReverse)(&idx, x) ? MI_SIZE_BITS - 1 - (size_t)idx : MI_SIZE_BITS);
|
||||
#elif mi_has_builtinz(clz)
|
||||
return (x!=0 ? (size_t)mi_builtinz(clz)(x) : MI_SIZE_BITS);
|
||||
#elif defined(__GNUC__) && (MI_ARCH_X64 || MI_ARCH_X86)
|
||||
if (x==0) return MI_SIZE_BITS;
|
||||
size_t r;
|
||||
__asm ("bsr\t%1, %0" : "=r"(r) : "r"(x) : "cc");
|
||||
return (MI_SIZE_BITS - 1 - r);
|
||||
#else
|
||||
#define MI_HAS_FAST_BITSCAN 0
|
||||
return (x!=0 ? _mi_clz_generic(x) : MI_SIZE_BITS);
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifndef MI_HAS_FAST_BITSCAN
|
||||
#define MI_HAS_FAST_BITSCAN 1
|
||||
#endif
|
||||
|
||||
/* --------------------------------------------------------------------------------
|
||||
find trailing/leading zero (bit scan forward/reverse)
|
||||
-------------------------------------------------------------------------------- */
|
||||
|
||||
// Bit scan forward: find the least significant bit that is set (i.e. count trailing zero's)
|
||||
// return false if `x==0` (with `*idx` undefined) and true otherwise,
|
||||
// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`).
|
||||
static inline bool mi_bsf(size_t x, size_t* idx) {
|
||||
#if defined(__GNUC__) && MI_ARCH_X64 && defined(__BMI1__) && (!defined(__clang_major__) || __clang_major__ >= 9)
|
||||
// on x64 the carry flag is set on zero which gives better codegen
|
||||
bool is_zero;
|
||||
__asm ( "tzcnt\t%2, %1" : "=@ccc"(is_zero), "=r"(*idx) : "r"(x) : "cc" );
|
||||
return !is_zero;
|
||||
#elif 0 && defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
|
||||
unsigned long i;
|
||||
return (mi_msc_builtinz(_BitScanForward)(&i, x) ? (*idx = (size_t)i, true) : false);
|
||||
#else
|
||||
return (x!=0 ? (*idx = mi_ctz(x), true) : false);
|
||||
#endif
|
||||
}
|
||||
|
||||
// Bit scan reverse: find the most significant bit that is set
|
||||
// return false if `x==0` (with `*idx` undefined) and true otherwise,
|
||||
// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`).
|
||||
static inline bool mi_bsr(size_t x, size_t* idx) {
|
||||
#if 0 && defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
|
||||
unsigned long i;
|
||||
return (mi_msc_builtinz(_BitScanReverse)(&i, x) ? (*idx = (size_t)i, true) : false);
|
||||
#else
|
||||
return (x!=0 ? (*idx = MI_SIZE_BITS - 1 - mi_clz(x), true) : false);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
/* --------------------------------------------------------------------------------
|
||||
rotate
|
||||
-------------------------------------------------------------------------------- */
|
||||
|
||||
static inline size_t mi_rotr(size_t x, size_t r) {
|
||||
#if (mi_has_builtin(rotateright64) && MI_SIZE_BITS==64)
|
||||
return mi_builtin(rotateright64)(x,r);
|
||||
#elif (mi_has_builtin(rotateright32) && MI_SIZE_BITS==32)
|
||||
return mi_builtin(rotateright32)(x,r);
|
||||
#elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_ARM64)
|
||||
return _rotr64(x, (int)r);
|
||||
#elif defined(_MSC_VER) && (MI_ARCH_X86 || MI_ARCH_ARM32)
|
||||
return _lrotr(x,(int)r);
|
||||
#else
|
||||
// The term `(-rshift)&(BITS-1)` is written instead of `BITS - rshift` to
|
||||
// avoid UB when `rshift==0`. See <https://blog.regehr.org/archives/1063>
|
||||
const unsigned int rshift = (unsigned int)(r) & (MI_SIZE_BITS-1);
|
||||
return ((x >> rshift) | (x << ((-rshift) & (MI_SIZE_BITS-1))));
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline size_t mi_rotl(size_t x, size_t r) {
|
||||
#if (mi_has_builtin(rotateleft64) && MI_SIZE_BITS==64)
|
||||
return mi_builtin(rotateleft64)(x,r);
|
||||
#elif (mi_has_builtin(rotateleft32) && MI_SIZE_BITS==32)
|
||||
return mi_builtin(rotateleft32)(x,r);
|
||||
#elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_ARM64)
|
||||
return _rotl64(x, (int)r);
|
||||
#elif defined(_MSC_VER) && (MI_ARCH_X86 || MI_ARCH_ARM32)
|
||||
return _lrotl(x, (int)r);
|
||||
#else
|
||||
// The term `(-rshift)&(BITS-1)` is written instead of `BITS - rshift` to
|
||||
// avoid UB when `rshift==0`. See <https://blog.regehr.org/archives/1063>
|
||||
const unsigned int rshift = (unsigned int)(r) & (MI_SIZE_BITS-1);
|
||||
return ((x << rshift) | (x >> ((-rshift) & (MI_SIZE_BITS-1))));
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline uint32_t mi_rotl32(uint32_t x, uint32_t r) {
|
||||
#if mi_has_builtin(rotateleft32)
|
||||
return mi_builtin(rotateleft32)(x,r);
|
||||
#elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
|
||||
return _lrotl(x, (int)r);
|
||||
#else
|
||||
// The term `(-rshift)&(BITS-1)` is written instead of `BITS - rshift` to
|
||||
// avoid UB when `rshift==0`. See <https://blog.regehr.org/archives/1063>
|
||||
const unsigned int rshift = (unsigned int)(r) & 31;
|
||||
return ((x << rshift) | (x >> ((-rshift) & 31)));
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
#endif // MI_BITS_H
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,542 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#pragma once
|
||||
#ifndef MIMALLOC_PRIM_H
|
||||
#define MIMALLOC_PRIM_H
|
||||
#include "internal.h" // mi_decl_hidden
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// This file specifies the primitive portability API.
|
||||
// Each OS/host needs to implement these primitives, see `src/prim`
|
||||
// for implementations on Window, macOS, WASI, and Linux/Unix.
|
||||
//
|
||||
// note: on all primitive functions, we always have result parameters != NULL, and:
|
||||
// addr != NULL and page aligned
|
||||
// size > 0 and page aligned
|
||||
// the return value is an error code as an `int` where 0 is success
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
// OS memory configuration
|
||||
typedef struct mi_os_mem_config_s {
|
||||
size_t page_size; // default to 4KiB
|
||||
size_t large_page_size; // 0 if not supported, usually 2MiB (4MiB on Windows)
|
||||
size_t alloc_granularity; // smallest allocation size (usually 4KiB, on Windows 64KiB)
|
||||
size_t physical_memory_in_kib; // physical memory size in KiB
|
||||
size_t virtual_address_bits; // usually 48 or 56 bits on 64-bit systems. (used to determine secure randomization)
|
||||
bool has_overcommit; // can we reserve more memory than can be actually committed?
|
||||
bool has_partial_free; // can allocated blocks be freed partially? (true for mmap, false for VirtualAlloc)
|
||||
bool has_virtual_reserve; // supports virtual address space reservation? (if true we can reserve virtual address space without using commit or physical memory)
|
||||
bool has_transparent_huge_pages; // true if transparent huge pages are enabled (on Linux)
|
||||
} mi_os_mem_config_t;
|
||||
|
||||
// Initialize
|
||||
void _mi_prim_mem_init( mi_os_mem_config_t* config );
|
||||
|
||||
// Free OS memory
|
||||
int _mi_prim_free(void* addr, size_t size );
|
||||
|
||||
// Allocate OS memory. Return NULL on error.
|
||||
// The `try_alignment` is just a hint and the returned pointer does not have to be aligned.
|
||||
// If `commit` is false, the virtual memory range only needs to be reserved (with no access)
|
||||
// which will later be committed explicitly using `_mi_prim_commit`.
|
||||
// `is_zero` is set to true if the memory was zero initialized (as on most OS's)
|
||||
// The `hint_addr` address is either `NULL` or a preferred allocation address but can be ignored.
|
||||
// pre: !commit => !allow_large
|
||||
// try_alignment >= _mi_os_page_size() and a power of 2
|
||||
int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr);
|
||||
|
||||
// Commit memory. Returns error code or 0 on success.
|
||||
// For example, on Linux this would make the memory PROT_READ|PROT_WRITE.
|
||||
// `is_zero` is set to true if the memory was zero initialized (e.g. on Windows)
|
||||
int _mi_prim_commit(void* addr, size_t size, bool* is_zero);
|
||||
|
||||
// Decommit memory. Returns error code or 0 on success. The `needs_recommit` result is true
|
||||
// if the memory would need to be re-committed. For example, on Windows this is always true,
|
||||
// but on Linux we could use MADV_DONTNEED to decommit which does not need a recommit.
|
||||
// pre: needs_recommit != NULL
|
||||
int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit);
|
||||
|
||||
// Reset memory. The range keeps being accessible but the content might be reset to zero at any moment.
|
||||
// Returns error code or 0 on success.
|
||||
int _mi_prim_reset(void* addr, size_t size);
|
||||
|
||||
// Reuse memory. This is called for memory that is already committed but
|
||||
// may have been reset (`_mi_prim_reset`) or decommitted (`_mi_prim_decommit`) where `needs_recommit` was false.
|
||||
// Returns error code or 0 on success. On most platforms this is a no-op.
|
||||
int _mi_prim_reuse(void* addr, size_t size);
|
||||
|
||||
// Protect memory. Returns error code or 0 on success.
|
||||
int _mi_prim_protect(void* addr, size_t size, bool protect);
|
||||
|
||||
// Allocate huge (1GiB) pages possibly associated with a NUMA node.
|
||||
// `is_zero` is set to true if the memory was zero initialized (as on most OS's)
|
||||
// pre: size > 0 and a multiple of 1GiB.
|
||||
// numa_node is either negative (don't care), or a numa node number.
|
||||
int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr);
|
||||
|
||||
// Return the current NUMA node
|
||||
size_t _mi_prim_numa_node(void);
|
||||
|
||||
// Return the number of logical NUMA nodes
|
||||
size_t _mi_prim_numa_node_count(void);
|
||||
|
||||
// Clock ticks
|
||||
mi_msecs_t _mi_prim_clock_now(void);
|
||||
|
||||
// Return process information (only for statistics)
|
||||
typedef struct mi_process_info_s {
|
||||
mi_msecs_t elapsed;
|
||||
mi_msecs_t utime;
|
||||
mi_msecs_t stime;
|
||||
size_t current_rss;
|
||||
size_t peak_rss;
|
||||
size_t current_commit;
|
||||
size_t peak_commit;
|
||||
size_t page_faults;
|
||||
} mi_process_info_t;
|
||||
|
||||
void _mi_prim_process_info(mi_process_info_t* pinfo);
|
||||
|
||||
// Default stderr output. (only for warnings etc. with verbose enabled)
|
||||
// msg != NULL && _mi_strlen(msg) > 0
|
||||
void _mi_prim_out_stderr( const char* msg );
|
||||
|
||||
// Get an environment variable. (only for options)
|
||||
// name != NULL, result != NULL, result_size >= 64
|
||||
bool _mi_prim_getenv(const char* name, char* result, size_t result_size);
|
||||
|
||||
|
||||
// Fill a buffer with strong randomness; return `false` on error or if
|
||||
// there is no strong randomization available.
|
||||
bool _mi_prim_random_buf(void* buf, size_t buf_len);
|
||||
|
||||
// Called on the first thread start, and should ensure `_mi_thread_done` is called on thread termination.
|
||||
void _mi_prim_thread_init_auto_done(void);
|
||||
|
||||
// Called on process exit and may take action to clean up resources associated with the thread auto done.
|
||||
void _mi_prim_thread_done_auto_done(void);
|
||||
|
||||
// Called when the default theap for a thread changes
|
||||
void _mi_prim_thread_associate_default_theap(mi_theap_t* theap);
|
||||
|
||||
// Is this thread part of a thread pool?
|
||||
bool _mi_prim_thread_is_in_threadpool(void);
|
||||
|
||||
// Yield to other threads. Should be similar to `sleep(0)`.
|
||||
// Is called only in rare situations and does not have to be lightning fast.
|
||||
void _mi_prim_thread_yield(void);
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Access to TLS (thread local storage) slots.
|
||||
// We need fast access to both a unique thread id (in `free.c:mi_free`) and
|
||||
// to a thread-local theap pointer (in `alloc.c:mi_malloc`).
|
||||
// To achieve this we use specialized code for various platforms.
|
||||
//-------------------------------------------------------------------
|
||||
|
||||
// On some libc + platform combinations we can directly access a thread-local storage (TLS) slot.
|
||||
// The TLS layout depends on both the OS and libc implementation so we use specific tests for each main platform.
|
||||
// If you test on another platform and it works please send a PR :-)
|
||||
// see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register.
|
||||
//
|
||||
// Note: we would like to prefer `__builtin_thread_pointer()` nowadays instead of using assembly,
|
||||
// but unfortunately we can not detect support reliably (see issue #883)
|
||||
// We also use it on Apple OS as we use a TLS slot for the default theap there.
|
||||
#if (defined(_WIN32)) || \
|
||||
(defined(__GNUC__) && ( \
|
||||
(defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \
|
||||
|| (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__) || defined(__POWERPC__))) \
|
||||
|| (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \
|
||||
|| (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \
|
||||
|| (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \
|
||||
))
|
||||
|
||||
static inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept {
|
||||
void* res;
|
||||
const size_t ofs = (slot*sizeof(void*));
|
||||
#if defined(_WIN32)
|
||||
#if (_M_X64 || _M_AMD64) && !defined(_M_ARM64EC)
|
||||
res = (void*)__readgsqword((unsigned long)ofs); // direct load at offset from gs
|
||||
#elif _M_IX86 && !defined(_M_ARM64EC)
|
||||
res = (void*)__readfsdword((unsigned long)ofs); // direct load at offset from fs
|
||||
#else
|
||||
res = ((void**)NtCurrentTeb())[slot]; MI_UNUSED(ofs);
|
||||
#endif
|
||||
#elif defined(__i386__)
|
||||
__asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86 32-bit always uses GS
|
||||
#elif defined(__APPLE__) && defined(__x86_64__)
|
||||
__asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS
|
||||
#elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)
|
||||
__asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI
|
||||
#elif defined(__x86_64__)
|
||||
__asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS
|
||||
#elif defined(__arm__)
|
||||
void** tcb; MI_UNUSED(ofs);
|
||||
__asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
|
||||
res = tcb[slot];
|
||||
#elif defined(__aarch64__)
|
||||
void** tcb; MI_UNUSED(ofs);
|
||||
#if defined(__APPLE__) // M1, issue #343
|
||||
__asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb));
|
||||
#else
|
||||
__asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
|
||||
#endif
|
||||
res = tcb[slot];
|
||||
#elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781
|
||||
MI_UNUSED(ofs);
|
||||
res = pthread_getspecific(slot);
|
||||
#else
|
||||
#define MI_HAS_TLS_SLOT 0
|
||||
MI_UNUSED(ofs);
|
||||
res = NULL;
|
||||
#endif
|
||||
return res;
|
||||
}
|
||||
|
||||
#ifndef MI_HAS_TLS_SLOT
|
||||
#define MI_HAS_TLS_SLOT 1
|
||||
#endif
|
||||
|
||||
// setting a tls slot is only used on macOS for now
|
||||
static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexcept {
|
||||
const size_t ofs = (slot*sizeof(void*));
|
||||
#if defined(_WIN32)
|
||||
((void**)NtCurrentTeb())[slot] = value; MI_UNUSED(ofs);
|
||||
#elif defined(__i386__)
|
||||
__asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS
|
||||
#elif defined(__APPLE__) && defined(__x86_64__)
|
||||
__asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOS uses GS
|
||||
#elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)
|
||||
__asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI
|
||||
#elif defined(__x86_64__)
|
||||
__asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS
|
||||
#elif defined(__arm__)
|
||||
void** tcb; MI_UNUSED(ofs);
|
||||
__asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
|
||||
tcb[slot] = value;
|
||||
#elif defined(__aarch64__)
|
||||
void** tcb; MI_UNUSED(ofs);
|
||||
#if defined(__APPLE__) // M1, issue #343
|
||||
__asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb));
|
||||
#else
|
||||
__asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
|
||||
#endif
|
||||
tcb[slot] = value;
|
||||
#elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781
|
||||
MI_UNUSED(ofs);
|
||||
pthread_setspecific(slot, value);
|
||||
#else
|
||||
MI_UNUSED(ofs); MI_UNUSED(value);
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
// defined in `init.c`; do not use these directly
|
||||
extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_main; // theap belonging to the main heap
|
||||
extern mi_decl_hidden bool _mi_process_is_initialized; // has mi_process_init been called?
|
||||
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Get a fast unique thread id.
|
||||
//
|
||||
// Getting the thread id should be performant as it is called in the
|
||||
// fast path of `_mi_free` and we specialize for various platforms as
|
||||
// inlined definitions. Regular code should call `init.c:_mi_thread_id()`.
|
||||
// We only require _mi_prim_thread_id() to return a unique id
|
||||
// for each thread (unequal to zero).
|
||||
//-------------------------------------------------------------------
|
||||
|
||||
|
||||
// Do we have __builtin_thread_pointer? This would be the preferred way to get a unique thread id
|
||||
// but unfortunately, it seems we cannot test for this reliably at this time (see issue #883)
|
||||
// Nevertheless, it seems needed on older graviton platforms (see issue #851).
|
||||
// For now, we only enable this for specific platforms.
|
||||
#if !defined(MI_USE_BUILTIN_THREAD_POINTER) /* allow user override */
|
||||
#if !defined(__APPLE__) /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly (<https://github.com/microsoft/mimalloc/issues/343#issuecomment-763272369>)*/ \
|
||||
&& !defined(__CYGWIN__) \
|
||||
&& !defined(MI_LIBC_MUSL) \
|
||||
&& (!defined(__clang_major__) || __clang_major__ >= 14) /* older clang versions emit bad code; fall back to using the TLS slot (<https://lore.kernel.org/linux-arm-kernel/202110280952.352F66D8@keescook/T/>) */
|
||||
#if (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \
|
||||
|| (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \
|
||||
|| (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__)))
|
||||
#define MI_USE_BUILTIN_THREAD_POINTER 1
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
|
||||
static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept;
|
||||
|
||||
static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
|
||||
const mi_threadid_t tid = __mi_prim_thread_id();
|
||||
mi_assert_internal(tid > 1);
|
||||
mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0); // bottom 2 bits are clear?
|
||||
return tid;
|
||||
}
|
||||
|
||||
// Get a unique id for the current thread.
|
||||
#if defined(MI_PRIM_THREAD_ID)
|
||||
|
||||
static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
|
||||
const mi_threadid_t tid = MI_PRIM_THREAD_ID(); // used for example by CPython for a free threaded build (see python/cpython#115488)
|
||||
mi_assert_internal( (tid & 0x03) == 0 ); // mimalloc reserves the bottom 2 bits
|
||||
return tid;
|
||||
}
|
||||
|
||||
#elif defined(_WIN32)
|
||||
|
||||
static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
|
||||
// Windows: works on Intel and ARM in both 32- and 64-bit
|
||||
return (uintptr_t)NtCurrentTeb();
|
||||
}
|
||||
|
||||
#elif MI_USE_BUILTIN_THREAD_POINTER
|
||||
|
||||
static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
|
||||
// Works on most Unix based platforms with recent compilers
|
||||
return (uintptr_t)__builtin_thread_pointer();
|
||||
}
|
||||
|
||||
#elif MI_HAS_TLS_SLOT
|
||||
|
||||
static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
|
||||
#if defined(__BIONIC__)
|
||||
// issue #384, #495: on the Bionic libc (Android), slot 1 is the thread id
|
||||
// see: https://github.com/aosp-mirror/platform_bionic/blob/c44b1d0676ded732df4b3b21c5f798eacae93228/libc/platform/bionic/tls_defines.h#L86
|
||||
return (uintptr_t)mi_prim_tls_slot(1);
|
||||
#else
|
||||
// in all our other targets, slot 0 is the thread id
|
||||
// glibc: https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=sysdeps/x86_64/nptl/tls.h
|
||||
// apple: https://github.com/apple/darwin-xnu/blob/main/libsyscall/os/tsd.h#L36
|
||||
return (uintptr_t)mi_prim_tls_slot(0);
|
||||
#endif
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
// otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms).
|
||||
static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
|
||||
return (uintptr_t)&__mi_theap_main;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/* ----------------------------------------------------------------------------------------
|
||||
Get the thread local default theap: `_mi_theap_default()` (and the cached heap `_mi_theap_cached`).
|
||||
|
||||
This is inlined here as it is on the fast path for allocation functions.
|
||||
We have 4 models:
|
||||
|
||||
- MI_TLS_MODEL_THREAD_LOCAL: use regular thread local (default on Linux, FreeBSD, etc)
|
||||
On most platforms (Linux, FreeBSD, NetBSD, etc), this just returns a
|
||||
thread local variable (`__mi_theap_default`). With the initial-exec TLS model this ensures
|
||||
that the storage will always be available and properly initialized (with an empty theap).
|
||||
|
||||
On some platforms the underlying TLS implementation (or the loader) will call itself `malloc`
|
||||
on a first access to a thread local and recurse in the MI_TLS_MODEL_THREAD_LOCAL.
|
||||
A way around this is to define MI_TLS_RECURSE_GUARD which adds an extra check if the process
|
||||
is initialized before accessing the thread-local. This is a check in the fast path though
|
||||
so this should be avoided.
|
||||
|
||||
- MI_TLS_MODEL_FIXED_SLOT: use a fixed slot in the TLS block (default on macOS)
|
||||
This reserves an unused and fixed TLS slot. This is fast and avoids the problem
|
||||
where the underlying TLS implementation (or the loader) will call itself `malloc`
|
||||
on a first access to a thread local (and recurse in the MI_TLS_MODEL_THREAD_LOCAL).
|
||||
This goes wrong though if the OS or a library uses the same fixed slot.
|
||||
|
||||
- MI_TLS_MODEL_DYNAMIC_WIN32: use a dynamically allocated slot with TlsAlloc. (default on Windows)
|
||||
Windows has somewhat slow thread locals so by default we use TlsAlloc'd slots which
|
||||
can be more efficient. First tries to use one of the "direct" first 64 slots which
|
||||
are the fastest, but falls back to using "expansion" slots when needed (up to 1088 slots).
|
||||
(If the allocated slot happens to always be under 64 for a particular program,
|
||||
one might use cmake with `-DMI_WIN_DIRECT_TLS=ON` to skip the expansion slot test in the fast path.)
|
||||
|
||||
- MI_TLS_MODEL_DYNAMIC_PTHREADS: use `pthread_getspecific`. (default on OpenBSD, maybe good for Android as well?)
|
||||
Use pthread local storage. Somewhat slow but can work well depending on the platform.
|
||||
|
||||
Each model should define `MI_THEAP_INITASNULL` to signify that the initial value
|
||||
returned from `_mi_theap_default()` can be `NULL` (instead of the address of the empty heap).
|
||||
This incurs an extra check in the fast path (but can often be combined in an existing check).
|
||||
------------------------------------------------------------------------------------------- */
|
||||
|
||||
static inline mi_theap_t* _mi_theap_default(void);
|
||||
static inline mi_theap_t* _mi_theap_cached(void);
|
||||
|
||||
#if defined(_WIN32)
|
||||
#define MI_TLS_MODEL_DYNAMIC_WIN32 1
|
||||
#elif defined(__APPLE__) && MI_HAS_TLS_SLOT && !defined(__POWERPC__) // macOS on arm64 or x64
|
||||
// #define MI_TLS_MODEL_DYNAMIC_PTHREADS 1 // also works but a bit slower
|
||||
#define MI_TLS_MODEL_FIXED_SLOT 1
|
||||
#define MI_TLS_MODEL_FIXED_SLOT_DEFAULT 108 // seems unused. @apple: it would be great to get 2 official slots for custom allocators :-)
|
||||
#define MI_TLS_MODEL_FIXED_SLOT_CACHED 109
|
||||
// we used before __PTK_FRAMEWORK_OLDGC_KEY9 (89) but that seems used now.
|
||||
// see <https://github.com/rweichler/substrate/blob/master/include/pthread_machdep.h>
|
||||
#elif defined(__APPLE__) || defined(__OpenBSD__) || defined(__ANDROID__)
|
||||
#define MI_TLS_MODEL_DYNAMIC_PTHREADS 1
|
||||
// #define MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL 1
|
||||
#else
|
||||
#define MI_TLS_MODEL_THREAD_LOCAL 1
|
||||
#endif
|
||||
|
||||
// Declared this way to optimize register spills and branches
|
||||
mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void);
|
||||
|
||||
static inline mi_theap_t* __mi_theap_empty(void) {
|
||||
#if __GNUC__
|
||||
__asm(""); // prevent conditional load
|
||||
return (mi_theap_t*)&_mi_theap_empty;
|
||||
#else
|
||||
return _mi_theap_empty_get();
|
||||
#endif
|
||||
}
|
||||
|
||||
#if MI_TLS_MODEL_THREAD_LOCAL
|
||||
// Thread local with an initial value (default on Linux). Very efficient.
|
||||
|
||||
extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default; // default theap to allocate from
|
||||
extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached; // theap from the last used heap
|
||||
|
||||
static inline mi_theap_t* _mi_theap_default(void) {
|
||||
#if defined(MI_TLS_RECURSE_GUARD)
|
||||
if (mi_unlikely(!_mi_process_is_initialized)) return _mi_theap_empty_get();
|
||||
#endif
|
||||
return __mi_theap_default;
|
||||
}
|
||||
|
||||
static inline mi_theap_t* _mi_theap_cached(void) {
|
||||
return __mi_theap_cached;
|
||||
}
|
||||
|
||||
#elif MI_TLS_MODEL_FIXED_SLOT
|
||||
// Fixed TLS slot (default on macOS).
|
||||
#define MI_THEAP_INITASNULL 1
|
||||
|
||||
static inline mi_theap_t* _mi_theap_default(void) {
|
||||
return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_SLOT_DEFAULT);
|
||||
}
|
||||
|
||||
static inline mi_theap_t* _mi_theap_cached(void) {
|
||||
return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_SLOT_CACHED);
|
||||
}
|
||||
|
||||
#elif MI_TLS_MODEL_DYNAMIC_WIN32
|
||||
// Dynamic TLS slot (default on Windows)
|
||||
#define MI_THEAP_INITASNULL 1
|
||||
|
||||
// We try to use direct slots (64), but can also use the expansion slots (upto 1024 extra available)
|
||||
// See <https://www.geoffchappell.com/studies/windows/km/ntoskrnl/inc/api/pebteb/teb/index.htm> for the offsets.
|
||||
#if MI_SIZE_SIZE==4
|
||||
#define MI_TLS_EXPANSION_SLOT (0x0F94 / MI_SIZE_SIZE)
|
||||
#else
|
||||
#define MI_TLS_EXPANSION_SLOT (0x1780 / MI_SIZE_SIZE)
|
||||
#endif
|
||||
|
||||
extern mi_decl_hidden size_t _mi_theap_default_slot;
|
||||
extern mi_decl_hidden size_t _mi_theap_cached_slot;
|
||||
extern mi_decl_hidden size_t _mi_theap_default_expansion_slot;
|
||||
extern mi_decl_hidden size_t _mi_theap_cached_expansion_slot;
|
||||
|
||||
static inline mi_theap_t* _mi_theap_default(void) {
|
||||
const size_t slot = _mi_theap_default_slot;
|
||||
mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot);
|
||||
#if !MI_WIN_DIRECT_TLS
|
||||
if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ?
|
||||
if mi_likely(theap!=NULL) { // initialized (on this thread)?
|
||||
theap = ((mi_theap_t**)theap)[_mi_theap_default_expansion_slot];
|
||||
}
|
||||
}
|
||||
#endif
|
||||
return theap;
|
||||
}
|
||||
|
||||
static inline mi_theap_t* _mi_theap_cached(void) {
|
||||
const size_t slot = _mi_theap_cached_slot;
|
||||
mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot);
|
||||
#if !MI_WIN_DIRECT_TLS
|
||||
if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ?
|
||||
if mi_likely(theap!=NULL) { // initialized (on this thread)?
|
||||
theap = ((mi_theap_t**)theap)[_mi_theap_cached_expansion_slot];
|
||||
}
|
||||
}
|
||||
#endif
|
||||
return theap;
|
||||
}
|
||||
|
||||
#elif MI_TLS_MODEL_DYNAMIC_PTHREADS
|
||||
// Dynamic pthread slot on less common platforms. This is not too bad. (default on OpenBSD)
|
||||
#define MI_THEAP_INITASNULL 1
|
||||
|
||||
extern mi_decl_hidden pthread_key_t _mi_theap_default_key;
|
||||
extern mi_decl_hidden pthread_key_t _mi_theap_cached_key;
|
||||
|
||||
static inline mi_theap_t* _mi_theap_default(void) {
|
||||
#if !MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL
|
||||
// we can skip this check if using the initial key will return NULL from pthread_getspecific
|
||||
if mi_unlikely(_mi_theap_default_key==0) { return NULL; }
|
||||
#endif
|
||||
return (mi_theap_t*)pthread_getspecific(_mi_theap_default_key);
|
||||
}
|
||||
|
||||
static inline mi_theap_t* _mi_theap_cached(void) {
|
||||
#if !MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL
|
||||
// we can skip this check if using the initial key will return NULL from pthread_getspecific
|
||||
if mi_unlikely(_mi_theap_cached_key==0) { return NULL; }
|
||||
#endif
|
||||
return (mi_theap_t*)pthread_getspecific(_mi_theap_cached_key);
|
||||
}
|
||||
|
||||
#else
|
||||
#error "no TLS model is defined for this platform?"
|
||||
#endif
|
||||
|
||||
|
||||
// Check if a thread is initialized (without using a thread-local if using fixed slots)
|
||||
static inline bool _mi_thread_is_initialized(void) {
|
||||
return (mi_theap_is_initialized(_mi_theap_default()));
|
||||
}
|
||||
|
||||
// Get (and possible create) the theap belonging to a heap
|
||||
// We cache the last accessed theap in `_mi_theap_cached` for better performance.
|
||||
static inline mi_theap_t* _mi_heap_theap(const mi_heap_t* heap) {
|
||||
mi_theap_t* theap = _mi_theap_cached();
|
||||
#if MI_THEAP_INITASNULL
|
||||
if mi_likely(theap!=NULL && _mi_theap_heap(theap)==heap) return theap;
|
||||
#else
|
||||
if mi_likely(_mi_theap_heap(theap)==heap) return theap;
|
||||
#endif
|
||||
return _mi_heap_theap_get_or_init(heap);
|
||||
}
|
||||
|
||||
// Get the theap belonging to a heap without creating in if it is not yet initialized.
|
||||
static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap) {
|
||||
mi_theap_t* theap = _mi_theap_cached();
|
||||
#if MI_THEAP_INITASNULL
|
||||
if mi_unlikely(theap==NULL || _mi_theap_heap(theap)!=heap)
|
||||
#else
|
||||
if mi_unlikely(_mi_theap_heap(theap)!=heap)
|
||||
#endif
|
||||
{
|
||||
theap = _mi_heap_theap_get_peek(heap); // don't update the cache on a query (?)
|
||||
}
|
||||
mi_assert(theap==NULL || _mi_theap_heap(theap)==heap);
|
||||
return theap;
|
||||
}
|
||||
|
||||
// Find the associated theap or NULL if it does not exist (during shutdown)
|
||||
// Should be fast as it is called in `free.c:mi_free_try_collect`.
|
||||
static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page) {
|
||||
mi_heap_t* const heap = page->heap;
|
||||
mi_theap_t* theap;
|
||||
if mi_likely(heap==NULL) { theap = __mi_theap_main; } // note: on macOS accessing the thread_local can cause allocation during thread shutdown (and reinitialize the thread)!
|
||||
else { theap = _mi_heap_theap_peek(heap); }
|
||||
mi_assert_internal(theap==NULL || _mi_thread_id()==theap->tld->thread_id);
|
||||
return theap;
|
||||
}
|
||||
|
||||
#endif // MI_PRIM_H
|
||||
@@ -0,0 +1,150 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#pragma once
|
||||
#ifndef MI_TRACK_H
|
||||
#define MI_TRACK_H
|
||||
|
||||
/* ------------------------------------------------------------------------------------------------------
|
||||
Track memory ranges with macros for tools like Valgrind address sanitizer, or other memory checkers.
|
||||
These can be defined for tracking allocation:
|
||||
|
||||
#define mi_track_malloc_size(p,reqsize,size,zero)
|
||||
#define mi_track_free_size(p,_size)
|
||||
|
||||
The macros are set up such that the size passed to `mi_track_free_size`
|
||||
always matches the size of `mi_track_malloc_size`. (currently, `size == mi_usable_size(p)`).
|
||||
The `reqsize` is what the user requested, and `size >= reqsize`.
|
||||
The `size` is either byte precise (and `size==reqsize`) if `MI_PADDING` is enabled,
|
||||
or otherwise it is the usable block size which may be larger than the original request.
|
||||
Use `_mi_block_size_of(void* p)` to get the full block size that was allocated (including padding etc).
|
||||
The `zero` parameter is `true` if the allocated block is zero initialized.
|
||||
|
||||
Optional:
|
||||
|
||||
#define mi_track_align(p,alignedp,offset,size)
|
||||
#define mi_track_resize(p,oldsize,newsize)
|
||||
#define mi_track_init()
|
||||
|
||||
The `mi_track_align` is called right after a `mi_track_malloc` for aligned pointers in a block.
|
||||
The corresponding `mi_track_free` still uses the block start pointer and original size (corresponding to the `mi_track_malloc`).
|
||||
The `mi_track_resize` is currently unused but could be called on reallocations within a block.
|
||||
`mi_track_init` is called at program start.
|
||||
|
||||
The following macros are for tools like asan and valgrind to track whether memory is
|
||||
defined, undefined, or not accessible at all:
|
||||
|
||||
#define mi_track_mem_defined(p,size)
|
||||
#define mi_track_mem_undefined(p,size)
|
||||
#define mi_track_mem_noaccess(p,size)
|
||||
|
||||
-------------------------------------------------------------------------------------------------------*/
|
||||
|
||||
#if MI_TRACK_VALGRIND
|
||||
// valgrind tool
|
||||
|
||||
#define MI_TRACK_ENABLED 1
|
||||
#define MI_TRACK_HEAP_DESTROY 1 // track free of individual blocks on theap_destroy
|
||||
#define MI_TRACK_TOOL "valgrind"
|
||||
|
||||
#include <valgrind/valgrind.h>
|
||||
#include <valgrind/memcheck.h>
|
||||
|
||||
#define mi_track_malloc_size(p,reqsize,size,zero) VALGRIND_MALLOCLIKE_BLOCK(p,size,MI_PADDING_SIZE /*red zone*/,zero)
|
||||
#define mi_track_free_size(p,_size) VALGRIND_FREELIKE_BLOCK(p,MI_PADDING_SIZE /*red zone*/)
|
||||
#define mi_track_resize(p,oldsize,newsize) VALGRIND_RESIZEINPLACE_BLOCK(p,oldsize,newsize,MI_PADDING_SIZE /*red zone*/)
|
||||
#define mi_track_mem_defined(p,size) VALGRIND_MAKE_MEM_DEFINED(p,size)
|
||||
#define mi_track_mem_undefined(p,size) VALGRIND_MAKE_MEM_UNDEFINED(p,size)
|
||||
#define mi_track_mem_noaccess(p,size) VALGRIND_MAKE_MEM_NOACCESS(p,size)
|
||||
|
||||
#elif MI_TRACK_ASAN
|
||||
// address sanitizer
|
||||
|
||||
#define MI_TRACK_ENABLED 1
|
||||
#define MI_TRACK_HEAP_DESTROY 0
|
||||
#define MI_TRACK_TOOL "asan"
|
||||
|
||||
#include <sanitizer/asan_interface.h>
|
||||
|
||||
#define mi_track_malloc_size(p,reqsize,size,zero) ASAN_UNPOISON_MEMORY_REGION(p,size)
|
||||
#define mi_track_free_size(p,size) ASAN_POISON_MEMORY_REGION(p,size)
|
||||
#define mi_track_mem_defined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size)
|
||||
#define mi_track_mem_undefined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size)
|
||||
#define mi_track_mem_noaccess(p,size) ASAN_POISON_MEMORY_REGION(p,size)
|
||||
|
||||
#elif MI_TRACK_ETW
|
||||
// windows event tracing
|
||||
|
||||
#define MI_TRACK_ENABLED 1
|
||||
#define MI_TRACK_HEAP_DESTROY 1
|
||||
#define MI_TRACK_TOOL "ETW"
|
||||
|
||||
#include "../src/prim/windows/etw.h"
|
||||
|
||||
#define mi_track_init() EventRegistermicrosoft_windows_mimalloc()
|
||||
#define mi_track_done() EventUnregistermicrosoft_windows_mimalloc()
|
||||
#define mi_track_malloc_size(p,reqsize,size,zero) EventWriteETW_MI_ALLOC((UINT64)(p), size)
|
||||
#define mi_track_free_size(p,size) EventWriteETW_MI_FREE((UINT64)(p), size)
|
||||
|
||||
#else
|
||||
// no tracking
|
||||
|
||||
#define MI_TRACK_ENABLED 0
|
||||
#define MI_TRACK_HEAP_DESTROY 0
|
||||
#define MI_TRACK_TOOL "none"
|
||||
|
||||
#define mi_track_malloc_size(p,reqsize,size,zero)
|
||||
#define mi_track_free_size(p,_size)
|
||||
|
||||
#endif
|
||||
|
||||
// -------------------
|
||||
// Utility definitions
|
||||
|
||||
#ifndef mi_track_resize
|
||||
#define mi_track_resize(p,oldsize,newsize) do{ mi_track_free_size(p,oldsize); mi_track_malloc(p,newsize,false); } while(0)
|
||||
#endif
|
||||
|
||||
#ifndef mi_track_align
|
||||
#define mi_track_align(p,alignedp,offset,size) mi_track_mem_noaccess(p,offset)
|
||||
#endif
|
||||
|
||||
#ifndef mi_track_init
|
||||
#define mi_track_init()
|
||||
#endif
|
||||
|
||||
#ifndef mi_track_done
|
||||
#define mi_track_done()
|
||||
#endif
|
||||
|
||||
#ifndef mi_track_mem_defined
|
||||
#define mi_track_mem_defined(p,size)
|
||||
#endif
|
||||
|
||||
#ifndef mi_track_mem_undefined
|
||||
#define mi_track_mem_undefined(p,size)
|
||||
#endif
|
||||
|
||||
#ifndef mi_track_mem_noaccess
|
||||
#define mi_track_mem_noaccess(p,size)
|
||||
#endif
|
||||
|
||||
|
||||
#if MI_PADDING
|
||||
#define mi_track_malloc(p,reqsize,zero) \
|
||||
if ((p)!=NULL) { \
|
||||
mi_assert_internal(mi_usable_size(p)==(reqsize)); \
|
||||
mi_track_malloc_size(p,reqsize,reqsize,zero); \
|
||||
}
|
||||
#else
|
||||
#define mi_track_malloc(p,reqsize,zero) \
|
||||
if ((p)!=NULL) { \
|
||||
mi_assert_internal(mi_usable_size(p)>=(reqsize)); \
|
||||
mi_track_malloc_size(p,reqsize,mi_usable_size(p),zero); \
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // MI_TRACK_H
|
||||
@@ -0,0 +1,732 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#pragma once
|
||||
#ifndef MI_TYPES_H
|
||||
#define MI_TYPES_H
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// This file contains the main type definitions for mimalloc:
|
||||
// mi_heap_t : all data for a heap; usually there is just one main default heap.
|
||||
// mi_theap_t : a thread local heap belonging to a specific heap:
|
||||
// maintains lists of thread-local heap pages that have free space.
|
||||
// mi_page_t : a mimalloc page (usually 64KiB or 512KiB) from
|
||||
// where objects of a single size are allocated.
|
||||
// Note: we write "OS page" for OS memory pages while
|
||||
// using plain "page" for mimalloc pages (`mi_page_t`).
|
||||
// mi_arena_t : a large memory area where pages are allocated (process shared)
|
||||
// mi_tld_t : thread local data
|
||||
// mi_subproc_t : all heaps belong to a sub-process (usually just the main one)
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
|
||||
#include <mimalloc-stats.h>
|
||||
#include <stddef.h> // ptrdiff_t
|
||||
#include <stdint.h> // uintptr_t, uint16_t, etc
|
||||
#include <stdbool.h> // bool
|
||||
#include <limits.h> // SIZE_MAX etc.
|
||||
#include <errno.h> // error codes
|
||||
#include "bits.h" // size defines (MI_INTPTR_SIZE etc), bit operations
|
||||
#include "atomic.h" // _Atomic primitives
|
||||
|
||||
// Minimal alignment necessary. On most platforms 16 bytes are needed
|
||||
// due to SSE registers for example. This must be at least `sizeof(void*)`
|
||||
#ifndef MI_MAX_ALIGN_SIZE
|
||||
#define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t)
|
||||
#endif
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Variants
|
||||
// ------------------------------------------------------
|
||||
|
||||
// Define NDEBUG in the release version to disable assertions.
|
||||
// #define NDEBUG
|
||||
|
||||
// Define MI_TRACK_<tool> to enable tracking support
|
||||
// #define MI_TRACK_VALGRIND 1
|
||||
// #define MI_TRACK_ASAN 1
|
||||
// #define MI_TRACK_ETW 1
|
||||
|
||||
// Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance).
|
||||
// #define MI_STAT 1
|
||||
|
||||
// Define MI_SECURE to enable security mitigations
|
||||
// #define MI_SECURE 1 // guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption
|
||||
// #define MI_SECURE 2 // randomize relative allocation addresses (within mimalloc pages)
|
||||
// #define MI_SECURE 3 // encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free)
|
||||
// #define MI_SECURE 4 // checks for double free (may be more expensive) (`-DMI_SECURE=ON`)
|
||||
// #define MI_SECURE 5 // guard page at the end of each mimalloc page (expensive!) (`-DMI_SECURE_FULL=ON`)
|
||||
|
||||
#if !defined(MI_SECURE)
|
||||
#define MI_SECURE 0
|
||||
#endif
|
||||
|
||||
// Define MI_DEBUG for assertion and invariant checking
|
||||
// #define MI_DEBUG 1 // basic assertion checks and statistics, check double free, corrupted free list, and invalid pointer free. (cmake -DMI_DEBUG=ON)
|
||||
// #define MI_DEBUG 2 // + internal assertion checks (cmake -DMI_DEBUG_INTERNAL=ON)
|
||||
// #define MI_DEBUG 3 // + extensive internal invariant checking (cmake -DMI_DEBUG_FULL=ON)
|
||||
#if !defined(MI_DEBUG)
|
||||
#if defined(MI_BUILD_RELEASE) || defined(NDEBUG)
|
||||
#define MI_DEBUG 0
|
||||
#else
|
||||
#define MI_DEBUG 2
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// Statistics (0=only essential, 1=normal, 2=more fine-grained (expensive) tracking)
|
||||
#ifndef MI_STAT
|
||||
#if (MI_DEBUG>0)
|
||||
#define MI_STAT 2
|
||||
#else
|
||||
#define MI_STAT 0
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// Enable guard pages behind objects of a certain size (set by the MIMALLOC_GUARDED_MIN/MAX/SAMPLE_RATE options)
|
||||
#if !defined(MI_GUARDED) && MI_DEBUG && !defined(NDEBUG) && !MI_PAGE_META_ALIGNED_FREE_SMALL
|
||||
#define MI_GUARDED 1
|
||||
#endif
|
||||
|
||||
// Reserve extra padding at the end of each block to be more resilient against theap block overflows.
|
||||
// The padding can detect buffer overflow on free.
|
||||
#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW))
|
||||
#define MI_PADDING 1
|
||||
#endif
|
||||
|
||||
// Check padding bytes; allows byte-precise buffer overflow detection
|
||||
#if !defined(MI_PADDING_CHECK) && MI_PADDING && (MI_SECURE>=3 || MI_DEBUG>=1)
|
||||
#define MI_PADDING_CHECK 1
|
||||
#endif
|
||||
|
||||
|
||||
// Encoded free lists allow detection of corrupted free lists
|
||||
// and can detect buffer overflows, modify after free, and double `free`s.
|
||||
#if (MI_SECURE>=3 || MI_DEBUG>=1)
|
||||
#define MI_ENCODE_FREELIST 1
|
||||
#endif
|
||||
|
||||
// Enable large pages for objects between 64KiB and 512KiB.
|
||||
// This should perhaps be disabled by default as for many workloads the block sizes above 64 KiB
|
||||
// are quite random which can lead to too many partially used large pages (but see issue #1104).
|
||||
#ifndef MI_ENABLE_LARGE_PAGES
|
||||
#define MI_ENABLE_LARGE_PAGES 1
|
||||
#endif
|
||||
|
||||
// Place page meta info at the start of the page area or keep it separate?
|
||||
// Separate keeps the page info at the arena start (default) which is more secure
|
||||
// and reduces wasted space due to alignment and block sizes.
|
||||
// (but also reserves more memory up front (about 2MiB per GiB))
|
||||
#if !defined(MI_PAGE_META_IS_SEPARATED)
|
||||
#if MI_PAGE_MAP_FLAT
|
||||
#define MI_PAGE_META_IS_SEPARATED 0
|
||||
#else
|
||||
#define MI_PAGE_META_IS_SEPARATED 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// We can choose to only put page info of small pages at the start of the page area.
|
||||
// This can be used to have a slightly faster `mi_free_small` function for specialized
|
||||
// cases (like language runtime systems).
|
||||
#if !defined(MI_PAGE_META_ALIGNED_FREE_SMALL)
|
||||
#define MI_PAGE_META_ALIGNED_FREE_SMALL 0
|
||||
#endif
|
||||
|
||||
// Configuration checks
|
||||
#if !MI_PAGE_META_IS_SEPARATED && MI_SECURE
|
||||
#error "secure mode should use separated page infos"
|
||||
#endif
|
||||
#if MI_PAGE_META_ALIGNED_FREE_SMALL && MI_SECURE
|
||||
#error "secure mode cannot use MI_PAGE_META_ALIGNED_FREE_SMALL"
|
||||
#endif
|
||||
#if MI_PAGE_META_IS_SEPARATED && MI_PAGE_MAP_FLAT
|
||||
#error "cannot have a flat page map with separated page infos"
|
||||
#endif
|
||||
#if MI_DEBUG && NDEBUG
|
||||
#warning "mimalloc assertions enabled in a release build"
|
||||
#endif
|
||||
|
||||
|
||||
// --------------------------------------------------------------
|
||||
// Sizes of internal data-structures
|
||||
// (comments specify sizes on 64-bit, usually 32-bit is halved)
|
||||
// --------------------------------------------------------------
|
||||
|
||||
// Main size parameter; determines max arena sizes and max arena object sizes etc.
|
||||
#ifndef MI_ARENA_SLICE_SHIFT
|
||||
#ifdef MI_SMALL_PAGE_SHIFT // backward compatibility
|
||||
#define MI_ARENA_SLICE_SHIFT MI_SMALL_PAGE_SHIFT
|
||||
#elif MI_SECURE>=5 && __APPLE__ && MI_ARCH_ARM64
|
||||
#define MI_ARENA_SLICE_SHIFT (17) // 128 KiB to not waste too much due to 16 KiB guard pages
|
||||
#else
|
||||
#define MI_ARENA_SLICE_SHIFT (13 + MI_SIZE_SHIFT) // 64 KiB (32 KiB on 32-bit)
|
||||
#endif
|
||||
#endif
|
||||
#if MI_ARENA_SLICE_SHIFT < 12
|
||||
#error Arena slices should be at least 4KiB
|
||||
#endif
|
||||
|
||||
#ifndef MI_BCHUNK_BITS_SHIFT
|
||||
#if MI_ARENA_SLICE_SHIFT <= 13 // <= 8KiB
|
||||
#define MI_BCHUNK_BITS_SHIFT (7) // 128 bits
|
||||
#elif MI_ARENA_SLICE_SHIFT < 16 // <= 32KiB
|
||||
#define MI_BCHUNK_BITS_SHIFT (8) // 256 bits
|
||||
#else
|
||||
#define MI_BCHUNK_BITS_SHIFT (6 + MI_SIZE_SHIFT) // 512 bits (or 256 on 32-bit)
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#define MI_BCHUNK_BITS (1 << MI_BCHUNK_BITS_SHIFT) // sub-bitmaps in arena's are "bchunks" of 512 bits
|
||||
#define MI_ARENA_SLICE_SIZE (MI_ZU(1) << MI_ARENA_SLICE_SHIFT) // arena's allocate in slices of 64 KiB
|
||||
#define MI_ARENA_SLICE_ALIGN (MI_ARENA_SLICE_SIZE)
|
||||
|
||||
#define MI_ARENA_MIN_OBJ_SLICES (1)
|
||||
#define MI_ARENA_MAX_CHUNK_OBJ_SLICES (MI_BCHUNK_BITS) // 32 MiB (or 8 MiB on 32-bit)
|
||||
|
||||
#define MI_ARENA_MIN_OBJ_SIZE (MI_ARENA_MIN_OBJ_SLICES * MI_ARENA_SLICE_SIZE)
|
||||
#define MI_ARENA_MAX_CHUNK_OBJ_SIZE (MI_ARENA_MAX_CHUNK_OBJ_SLICES * MI_ARENA_SLICE_SIZE)
|
||||
|
||||
#if MI_ARENA_MAX_CHUNK_OBJ_SIZE < MI_SIZE_SIZE*1024
|
||||
#error maximum object size may be too small to hold local thread data
|
||||
#endif
|
||||
|
||||
#define MI_SMALL_PAGE_SIZE MI_ARENA_MIN_OBJ_SIZE // 64 KiB
|
||||
#define MI_MEDIUM_PAGE_SIZE (8*MI_SMALL_PAGE_SIZE) // 512 KiB (=byte in the bchunk bitmap)
|
||||
#define MI_LARGE_PAGE_SIZE (MI_SIZE_SIZE*MI_MEDIUM_PAGE_SIZE) // 4 MiB (=word in the bchunk bitmap)
|
||||
|
||||
|
||||
// Maximum number of size classes. (spaced exponentially in 12.5% increments)
|
||||
#if MI_BIN_HUGE != 73U
|
||||
#error "mimalloc internal: expecting 73 bins"
|
||||
#endif
|
||||
#define MI_BIN_FULL (MI_BIN_HUGE+1)
|
||||
#define MI_BIN_COUNT (MI_BIN_FULL+1)
|
||||
|
||||
// We never allocate more than PTRDIFF_MAX (see also <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
|
||||
#define MI_MAX_ALLOC_SIZE PTRDIFF_MAX
|
||||
|
||||
// Minimal commit for a page on-demand commit (should be >= OS page size)
|
||||
#define MI_PAGE_MIN_COMMIT_SIZE MI_ARENA_SLICE_SIZE
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Arena's are large reserved areas of memory allocated from
|
||||
// the OS that are managed by mimalloc to efficiently
|
||||
// allocate MI_ARENA_SLICE_SIZE slices of memory for the
|
||||
// mimalloc pages.
|
||||
// ------------------------------------------------------
|
||||
|
||||
// A large memory arena where pages are allocated in.
|
||||
typedef struct mi_arena_s mi_arena_t; // defined below
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Heaps contain allocated blocks. Heaps are self-contained
|
||||
// but share the (sub-process) memory in the arena's.
|
||||
// ------------------------------------------------------
|
||||
|
||||
// A first-class heap.
|
||||
typedef struct mi_heap_s mi_heap_t; // heaps
|
||||
|
||||
// ------------------------------------------------------
|
||||
// We can have sub-processes that are fully separated
|
||||
// from each other (for running multiple Python interpreters
|
||||
// for example). A sub-process holds the memory arenas and heaps.
|
||||
// ------------------------------------------------------
|
||||
|
||||
// A sub-process
|
||||
typedef struct mi_subproc_s mi_subproc_t;
|
||||
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// a memory id tracks the provenance of arena/OS allocated memory
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
// Memory can reside in arena's, direct OS allocated, meta-data pages, or statically allocated.
|
||||
// The memid keeps track of this.
|
||||
typedef enum mi_memkind_e {
|
||||
MI_MEM_NONE, // not allocated
|
||||
MI_MEM_EXTERNAL, // not owned by mimalloc but provided externally (via `mi_manage_os_memory` for example)
|
||||
MI_MEM_STATIC, // allocated in a static area and should not be freed (the initial main theap data for example (`init.c`))
|
||||
MI_MEM_META, // allocated with the meta data allocator (`arena-meta.c`)
|
||||
MI_MEM_OS, // allocated from the OS
|
||||
MI_MEM_OS_HUGE, // allocated as huge OS pages (usually 1GiB, pinned to physical memory)
|
||||
MI_MEM_OS_REMAP, // allocated in a remapable area (i.e. using `mremap`)
|
||||
MI_MEM_ARENA, // allocated from an arena (the usual case) (`arena.c`)
|
||||
MI_MEM_HEAP_MAIN // allocated in the main heap (for theaps)
|
||||
} mi_memkind_t;
|
||||
|
||||
static inline bool mi_memkind_is_os(mi_memkind_t memkind) {
|
||||
return (memkind >= MI_MEM_OS && memkind <= MI_MEM_OS_REMAP);
|
||||
}
|
||||
|
||||
static inline bool mi_memkind_needs_no_free(mi_memkind_t memkind) {
|
||||
return (memkind <= MI_MEM_STATIC);
|
||||
}
|
||||
|
||||
|
||||
typedef struct mi_memid_os_info {
|
||||
void* base; // actual base address of the block (used for offset aligned allocations)
|
||||
size_t size; // allocated full size
|
||||
// size_t alignment; // alignment at allocation
|
||||
} mi_memid_os_info_t;
|
||||
|
||||
typedef struct mi_memid_arena_info {
|
||||
mi_arena_t* arena; // arena that contains this memory
|
||||
uint32_t slice_index; // slice index in the arena
|
||||
uint32_t slice_count; // allocated slices
|
||||
} mi_memid_arena_info_t;
|
||||
|
||||
typedef struct mi_memid_meta_info {
|
||||
void* meta_page; // meta-page that contains the block
|
||||
uint32_t block_index; // block index in the meta-data page
|
||||
uint32_t block_count; // allocated blocks
|
||||
} mi_memid_meta_info_t;
|
||||
|
||||
typedef struct mi_memid_s {
|
||||
union {
|
||||
mi_memid_os_info_t os; // only used for MI_MEM_OS
|
||||
mi_memid_arena_info_t arena; // only used for MI_MEM_ARENA
|
||||
mi_memid_meta_info_t meta; // only used for MI_MEM_META
|
||||
} mem;
|
||||
mi_memkind_t memkind;
|
||||
bool is_pinned; // `true` if we cannot decommit/reset/protect in this memory (e.g. when allocated using large (2Mib) or huge (1GiB) OS pages)
|
||||
bool initially_committed;// `true` if the memory was originally allocated as committed
|
||||
bool initially_zero; // `true` if the memory was originally zero initialized
|
||||
} mi_memid_t;
|
||||
|
||||
|
||||
static inline bool mi_memid_is_os(mi_memid_t memid) {
|
||||
return mi_memkind_is_os(memid.memkind);
|
||||
}
|
||||
|
||||
static inline bool mi_memid_needs_no_free(mi_memid_t memid) {
|
||||
return mi_memkind_needs_no_free(memid.memkind);
|
||||
}
|
||||
|
||||
static inline mi_arena_t* mi_memid_arena(mi_memid_t memid) {
|
||||
return (memid.memkind == MI_MEM_ARENA ? memid.mem.arena.arena : NULL);
|
||||
}
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Mimalloc pages contain allocated blocks
|
||||
// ------------------------------------------------------
|
||||
|
||||
// The free lists use encoded next fields
|
||||
// (Only actually encodes when MI_ENCODED_FREELIST is defined.)
|
||||
typedef uintptr_t mi_encoded_t;
|
||||
|
||||
// thread id's
|
||||
typedef size_t mi_threadid_t;
|
||||
|
||||
// free lists contain blocks
|
||||
typedef struct mi_block_s {
|
||||
mi_encoded_t next;
|
||||
} mi_block_t;
|
||||
|
||||
|
||||
// The page flags are put in the bottom 2 bits of the thread_id (for a fast test in `mi_free`)
|
||||
// If `has_interior_pointers` is true if the page has pointers at an offset in a block (so we have to unalign to the block start before free-ing)
|
||||
// `in_full_queue` is true if the page is full and resides in the full queue (so we move it to a regular queue on free-ing)
|
||||
#define MI_PAGE_IN_FULL_QUEUE MI_ZU(0x01)
|
||||
#define MI_PAGE_HAS_INTERIOR_POINTERS MI_ZU(0x02)
|
||||
#define MI_PAGE_FLAG_MASK MI_ZU(0x03)
|
||||
typedef size_t mi_page_flags_t;
|
||||
|
||||
// There are two special threadid's: 0 for pages that are abandoned (and not in a theap queue),
|
||||
// and 4 for abandoned & mapped threads -- abandoned-mapped pages are abandoned but also mapped
|
||||
// in an arena (in `mi_heap_t.arena_pages.pages_abandoned`) so these can be quickly found for reuse.
|
||||
// Abandoning partially used pages allows for sharing of this memory between threads (in particular if threads are blocked)
|
||||
#define MI_THREADID_ABANDONED MI_ZU(0)
|
||||
#define MI_THREADID_ABANDONED_MAPPED (MI_PAGE_FLAG_MASK + 1)
|
||||
|
||||
// Thread free list.
|
||||
// Points to a list of blocks that are freed by other threads.
|
||||
// The least-bit is set if the page is owned by the current thread. (`mi_page_is_owned`).
|
||||
// Ownership is required before we can read any non-atomic fields in the page.
|
||||
// This way we can push a block on the thread free list and try to claim ownership atomically in `free.c:mi_free_block_mt`.
|
||||
typedef uintptr_t mi_thread_free_t;
|
||||
|
||||
// A page contains blocks of one specific size (`block_size`).
|
||||
// Each page has three list of free blocks:
|
||||
// `free` for blocks that can be allocated,
|
||||
// `local_free` for freed blocks that are not yet available to `mi_malloc`
|
||||
// `thread_free` for freed blocks by other threads
|
||||
// The `local_free` and `thread_free` lists are migrated to the `free` list
|
||||
// when it is exhausted. The separate `local_free` list is necessary to
|
||||
// implement a monotonic heartbeat. The `thread_free` list is needed for
|
||||
// avoiding atomic operations when allocating from the owning thread.
|
||||
//
|
||||
// `used - |thread_free|` == actual blocks that are in use (alive)
|
||||
// `used - |thread_free| + |free| + |local_free| == capacity`
|
||||
//
|
||||
// We don't count "freed" (as |free|) but use only the `used` field to reduce
|
||||
// the number of memory accesses in the `mi_page_all_free` function(s).
|
||||
// Use `_mi_page_free_collect` to collect the thread_free list and update the `used` count.
|
||||
//
|
||||
// Notes:
|
||||
// - Non-atomic fields can only be accessed if having _ownership_ (low bit of `xthread_free` is 1).
|
||||
// Combining the `thread_free` list with an ownership bit allows a concurrent `free` to atomically
|
||||
// free an object and (re)claim ownership if the page was abandoned.
|
||||
// - If a page is not part of a theap it is called "abandoned" (`theap==NULL`) -- in
|
||||
// that case the `xthreadid` is 0 or 4 (4 is for abandoned pages that
|
||||
// are in the `pages_abandoned` lists of an arena, these are called "mapped" abandoned pages).
|
||||
// - page flags are in the bottom 3 bits of `xthread_id` for the fast path in `mi_free`.
|
||||
// - The layout is optimized for `free.c:mi_free` and `alloc.c:mi_page_alloc`
|
||||
// - Using `uint16_t` does not seem to slow things down
|
||||
|
||||
typedef struct mi_page_s {
|
||||
_Atomic(mi_threadid_t) xthread_id; // thread this page belongs to. (= `theap->thread_id (or 0 or 4 if abandoned) | page_flags`)
|
||||
|
||||
mi_block_t* free; // list of available free blocks (`malloc` allocates from this list)
|
||||
uint16_t used; // number of blocks in use (including blocks in `thread_free`)
|
||||
uint16_t capacity; // number of blocks committed
|
||||
uint16_t reserved; // number of blocks reserved in memory
|
||||
uint8_t retire_expire; // expiration count for retired blocks
|
||||
bool free_is_zero; // `true` if the blocks in the free list are zero initialized
|
||||
|
||||
mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`)
|
||||
_Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads (= `mi_block_t* | (1 if owned)`)
|
||||
|
||||
size_t block_size; // const: size available in each block (always `>0`)
|
||||
uint8_t* page_start; // const: start of the blocks
|
||||
|
||||
#if (MI_ENCODE_FREELIST || MI_PADDING)
|
||||
uintptr_t keys[2]; // const: two random keys to encode the free lists (see `_mi_block_next`) or padding canary
|
||||
#endif
|
||||
|
||||
mi_theap_t* theap; // the theap owning this page (may not be valid or NULL for abandoned pages)
|
||||
mi_heap_t* heap; // const: the heap owning this page
|
||||
|
||||
struct mi_page_s* next; // next page owned by the theap with the same `block_size`
|
||||
struct mi_page_s* prev; // previous page owned by the theap with the same `block_size`
|
||||
size_t slice_committed; // committed size relative to the first arena slice of the page data (or 0 if the page is fully committed already)
|
||||
mi_memid_t memid; // const: provenance of the page memory
|
||||
} mi_page_t;
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Object sizes
|
||||
// ------------------------------------------------------
|
||||
|
||||
#define MI_PAGE_ALIGN MI_ARENA_SLICE_ALIGN // pages must be aligned on this for the page map.
|
||||
#define MI_PAGE_MIN_START_BLOCK_ALIGN MI_MAX_ALIGN_SIZE // minimal block alignment for the first block in a page (16b)
|
||||
#define MI_PAGE_MAX_START_BLOCK_ALIGN2 (4*MI_KiB) // maximal block alignment for "power of 2"-sized blocks (such that we guarantee natural alignment)
|
||||
#define MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4*MI_KiB) // also aligns any multiple of this size to avoid TLB misses.
|
||||
#define MI_PAGE_MAX_OVERALLOC_ALIGN MI_ARENA_SLICE_SIZE // (64 KiB) limit for which we overallocate in arena pages, beyond this use OS allocation
|
||||
|
||||
// The max object sizes are intended to not waste more than ~ 12.5% internally over the page sizes.
|
||||
#define MI_SMALL_MAX_OBJ_SIZE ((MI_SMALL_PAGE_SIZE-MI_PAGE_OSPAGE_BLOCK_ALIGN2)/6) // = 10 KiB
|
||||
#if MI_ENABLE_LARGE_PAGES
|
||||
#define MI_MEDIUM_MAX_OBJ_SIZE ((MI_MEDIUM_PAGE_SIZE-MI_PAGE_OSPAGE_BLOCK_ALIGN2)/6) // ~ 84 KiB
|
||||
#define MI_LARGE_MAX_OBJ_SIZE (MI_LARGE_PAGE_SIZE/8) // <= 512 KiB // note: this must be a nice power of 2 or we get rounding issues with `_mi_bin`
|
||||
#else
|
||||
#define MI_MEDIUM_MAX_OBJ_SIZE (MI_MEDIUM_PAGE_SIZE/8) // <= 64 KiB
|
||||
#define MI_LARGE_MAX_OBJ_SIZE MI_MEDIUM_MAX_OBJ_SIZE // note: this must be a nice power of 2 or we get rounding issues with `_mi_bin`
|
||||
#endif
|
||||
#define MI_LARGE_MAX_OBJ_WSIZE (MI_LARGE_MAX_OBJ_SIZE/MI_SIZE_SIZE)
|
||||
|
||||
#if (MI_LARGE_MAX_OBJ_WSIZE >= 655360)
|
||||
#error "mimalloc internal: define more bins"
|
||||
#endif
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Page kinds
|
||||
// ------------------------------------------------------
|
||||
|
||||
typedef enum mi_page_kind_e {
|
||||
MI_PAGE_SMALL, // small blocks go into 64KiB pages
|
||||
MI_PAGE_MEDIUM, // medium blocks go into 512KiB pages
|
||||
MI_PAGE_LARGE, // larger blocks go into 4MiB pages (if `MI_ENABLE_LARGE_PAGES==1`)
|
||||
MI_PAGE_SINGLETON // page containing a single block.
|
||||
// used for blocks `> MI_LARGE_MAX_OBJ_SIZE` or an alignment `> MI_PAGE_MAX_OVERALLOC_ALIGN`.
|
||||
} mi_page_kind_t;
|
||||
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// A "theap" is a thread local heap which owns pages.
|
||||
// (making them thread-local avoids atomic operations)
|
||||
//
|
||||
// All theaps belong to a (non-thread-local) heap.
|
||||
// A theap just owns a set of pages for allocation and
|
||||
// can only be allocate/reallocate from the thread that created it.
|
||||
// Freeing blocks can be done from any thread though.
|
||||
//
|
||||
// Per thread, there is always a default theap that belongs
|
||||
// to the default heap. It is initialized to statically
|
||||
// point initially to an empty theap to avoid initialization
|
||||
// checks in the fast path.
|
||||
// ------------------------------------------------------
|
||||
|
||||
// Thread local data
|
||||
typedef struct mi_tld_s mi_tld_t; // defined below
|
||||
|
||||
// Pages of a certain block size are held in a queue.
|
||||
typedef struct mi_page_queue_s {
|
||||
mi_page_t* first;
|
||||
mi_page_t* last;
|
||||
size_t count;
|
||||
size_t block_size;
|
||||
} mi_page_queue_t;
|
||||
|
||||
// Random context
|
||||
typedef struct mi_random_cxt_s {
|
||||
uint32_t input[16];
|
||||
uint32_t output[16];
|
||||
int output_available;
|
||||
bool weak;
|
||||
} mi_random_ctx_t;
|
||||
|
||||
|
||||
// In debug mode there is a padding structure at the end of the blocks to check for buffer overflows
|
||||
#if MI_PADDING
|
||||
typedef struct mi_padding_s {
|
||||
uint32_t canary; // encoded block value to check validity of the padding (in case of overflow)
|
||||
uint32_t delta; // padding bytes before the block. (mi_usable_size(p) - delta == exact allocated bytes)
|
||||
} mi_padding_t;
|
||||
#define MI_PADDING_SIZE (sizeof(mi_padding_t))
|
||||
#define MI_PADDING_WSIZE ((MI_PADDING_SIZE + MI_INTPTR_SIZE - 1) / MI_INTPTR_SIZE)
|
||||
#else
|
||||
#define MI_PADDING_SIZE 0
|
||||
#define MI_PADDING_WSIZE 0
|
||||
#endif
|
||||
|
||||
#define MI_PAGES_DIRECT (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE + 1)
|
||||
|
||||
|
||||
// A thread-local heap ("theap") owns a set of thread-local pages.
|
||||
struct mi_theap_s {
|
||||
mi_tld_t* tld; // thread-local data
|
||||
_Atomic(mi_heap_t*) heap; // the heap this theap belongs to.
|
||||
_Atomic(size_t) refcount; // reference count
|
||||
unsigned long long heartbeat; // monotonic heartbeat count
|
||||
uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`)
|
||||
mi_random_ctx_t random; // random number context used for secure allocation
|
||||
size_t page_count; // total number of pages in the `pages` queues.
|
||||
size_t page_retired_min; // smallest retired index (retired pages are fully free, but still in the page queues)
|
||||
size_t page_retired_max; // largest retired index into the `pages` array.
|
||||
size_t pages_full_size; // optimization: total size of blocks in the pages of the full queue (issue #1220)
|
||||
long generic_count; // how often is `_mi_malloc_generic` called?
|
||||
long generic_collect_count; // how often is `_mi_malloc_generic` called without collecting?
|
||||
|
||||
mi_theap_t* tnext; // list of theaps in this thread
|
||||
mi_theap_t* tprev;
|
||||
mi_theap_t* hnext; // list of theaps of the owning `heap`
|
||||
mi_theap_t* hprev;
|
||||
|
||||
long page_full_retain; // how many full pages can be retained per queue (before abandoning them)
|
||||
bool allow_page_reclaim; // `true` if this theap should not reclaim abandoned pages
|
||||
bool allow_page_abandon; // `true` if this theap can abandon pages to reduce memory footprint
|
||||
#if MI_GUARDED
|
||||
size_t guarded_size_min; // minimal size for guarded objects
|
||||
size_t guarded_size_max; // maximal size for guarded objects
|
||||
size_t guarded_sample_rate; // sample rate (set to 0 to disable guarded pages)
|
||||
size_t guarded_sample_count; // current sample count (counting down to 0)
|
||||
#endif
|
||||
mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size.
|
||||
mi_page_queue_t pages[MI_BIN_COUNT]; // queue of pages for each size class (or "bin")
|
||||
mi_memid_t memid; // provenance of the theap struct itself (meta or os)
|
||||
mi_stats_t stats; // thread-local statistics
|
||||
};
|
||||
|
||||
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Heaps contain allocated blocks. Heaps are self-contained
|
||||
// but share the (sub-process) memory in the arena's.
|
||||
// ------------------------------------------------------
|
||||
|
||||
// Keep track of all owned and abandoned pages in the arena's
|
||||
struct mi_arena_pages_s;
|
||||
typedef struct mi_arena_pages_s mi_arena_pages_t;
|
||||
|
||||
#define MI_MAX_ARENAS (160) // Limited for now (and takes up .bss).. but arena's scale up exponentially (see `mi_arena_reserve`)
|
||||
// 160 arenas is enough for ~2 TiB memory
|
||||
|
||||
// A dynamic thread-local variable; 0 for an invalid thread-local
|
||||
typedef size_t mi_thread_local_t;
|
||||
|
||||
typedef struct mi_heap_s {
|
||||
mi_subproc_t* subproc; // a heap belongs to a subprocess
|
||||
size_t heap_seq; // unique sequence number for heaps in this subprocess
|
||||
mi_heap_t* next; // list of heaps in this subprocess
|
||||
mi_heap_t* prev;
|
||||
mi_thread_local_t theap; // dynamic thread local for the thread-local theaps of this heap
|
||||
|
||||
mi_arena_t* exclusive_arena; // if the heap should only allocate from a specific arena (or NULL)
|
||||
int numa_node; // if >=0, prefer this numa node for allocations
|
||||
|
||||
mi_theap_t* theaps; // list of all thread-local theaps belonging to this heap (using the `hnext`/`hprev` fields)
|
||||
mi_lock_t theaps_lock; // lock for the theaps list operations
|
||||
|
||||
_Atomic(size_t) abandoned_count[MI_BIN_COUNT]; // total count of abandoned pages in this heap
|
||||
mi_page_t* os_abandoned_pages; // list of pages that are OS allocated and not in an arena
|
||||
mi_lock_t os_abandoned_pages_lock; // lock for the os abandoned pages list (this lock protects list operations)
|
||||
|
||||
_Atomic(mi_arena_pages_t*) arena_pages[MI_MAX_ARENAS]; // track owned and abandoned pages in the arenas (entries can be NULL)
|
||||
mi_lock_t arena_pages_lock; // lock to update the arena_pages array
|
||||
|
||||
mi_stats_t stats; // statistics for this heap; periodically updated by merging from each theap
|
||||
} mi_heap_t;
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Sub processes do not reclaim or visit pages from other sub processes.
|
||||
// These are essentially the static variables of a process, and
|
||||
// usually there is only one subprocess. This can be used for example
|
||||
// by CPython to have separate interpreters within one process.
|
||||
// Each thread can only belong to one subprocess
|
||||
// (and needs to call `mi_subproc_add_current_thread` before any allocations).
|
||||
// ------------------------------------------------------
|
||||
|
||||
struct mi_subproc_s {
|
||||
size_t subproc_seq; // unique id for sub-processes
|
||||
mi_subproc_t* next; // list of all sub-processes
|
||||
mi_subproc_t* prev;
|
||||
|
||||
_Atomic(size_t) arena_count; // current count of arena's
|
||||
_Atomic(mi_arena_t*) arenas[MI_MAX_ARENAS]; // arena's of this sub-process
|
||||
mi_lock_t arena_reserve_lock; // lock to ensure arena's get reserved one at a time
|
||||
mi_decl_align(8) // needed on some 32-bit platforms
|
||||
_Atomic(int64_t) purge_expire; // expiration is set if any arenas can be purged
|
||||
|
||||
_Atomic(mi_heap_t*) heap_main; // main heap for this sub process
|
||||
mi_heap_t* heaps; // heaps belonging to this sub-process
|
||||
mi_lock_t heaps_lock;
|
||||
|
||||
_Atomic(size_t) thread_count; // current threads associated with this sub-process
|
||||
_Atomic(size_t) thread_total_count; // total created threads associated with this sub-process
|
||||
_Atomic(size_t) heap_count; // current heaps in this sub-process (== |heaps|)
|
||||
_Atomic(size_t) heap_total_count; // total created heaps in this sub-process
|
||||
|
||||
mi_memid_t memid; // provenance of this memory block (meta or static)
|
||||
mi_decl_align(8) // needed on some 32-bit platforms
|
||||
mi_stats_t stats; // subprocess statistics; updated for arena/OS stats like committed,
|
||||
// and otherwise merged with heap stats when those are deleted
|
||||
};
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Thread Local data
|
||||
// ------------------------------------------------------
|
||||
|
||||
// Milliseconds as in `int64_t` to avoid overflows
|
||||
typedef int64_t mi_msecs_t;
|
||||
|
||||
// Thread local data
|
||||
struct mi_tld_s {
|
||||
mi_threadid_t thread_id; // thread id of this thread
|
||||
size_t thread_seq; // thread sequence id (linear count of created threads)
|
||||
int numa_node; // thread preferred numa node
|
||||
mi_subproc_t* subproc; // sub-process this thread belongs to.
|
||||
mi_theap_t* theaps; // list of theaps in this thread (so we can abandon all when the thread terminates)
|
||||
mi_lock_t theaps_lock; // lock as the theaps list is sometimes accessed from another thread (on `mi_heap_free`)
|
||||
bool recurse; // true if deferred was called; used to prevent infinite recursion.
|
||||
bool is_in_threadpool; // true if this thread is part of a threadpool (and can run arbitrary tasks)
|
||||
mi_memid_t memid; // provenance of the tld memory itself (meta or OS)
|
||||
};
|
||||
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
Arenas are fixed area's of OS memory from which we can allocate
|
||||
large blocks (>= MI_ARENA_MIN_BLOCK_SIZE).
|
||||
In contrast to the rest of mimalloc, the arenas are shared between
|
||||
threads and need to be accessed using atomic operations (using atomic `mi_bitmap_t`'s).
|
||||
|
||||
Arenas are also used to for huge OS page (1GiB) reservations or for reserving
|
||||
OS memory upfront which can be improve performance or is sometimes needed
|
||||
on embedded devices. We can also employ this with WASI or `sbrk` systems
|
||||
to reserve large arenas upfront and be able to reuse the memory more effectively.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
#define MI_ARENA_BIN_COUNT (MI_BIN_COUNT)
|
||||
#define MI_ARENA_MIN_SIZE (MI_BCHUNK_BITS * MI_ARENA_SLICE_SIZE) // 32 MiB (or 8 MiB on 32-bit)
|
||||
#define MI_ARENA_MAX_SIZE (MI_BITMAP_MAX_BIT_COUNT * MI_ARENA_SLICE_SIZE)
|
||||
|
||||
typedef struct mi_bitmap_s mi_bitmap_t; // atomic bitmap (defined in `src/bitmap.h`)
|
||||
typedef struct mi_bbitmap_s mi_bbitmap_t; // atomic binned bitmap (defined in `src/bitmap.h`)
|
||||
|
||||
typedef struct mi_arena_pages_s {
|
||||
mi_bitmap_t* pages; // all registered pages (abandoned and owned)
|
||||
mi_bitmap_t* pages_abandoned[MI_ARENA_BIN_COUNT]; // abandoned pages per size bin (a set bit means the start of the page)
|
||||
// followed by the bitmaps (whose siz`es depend on the arena size)
|
||||
} mi_arena_pages_t;
|
||||
|
||||
|
||||
// A memory arena
|
||||
typedef struct mi_arena_s {
|
||||
mi_memid_t memid; // provenance of the memory area
|
||||
mi_subproc_t* subproc; // subprocess this arena belongs to (`this 'element-of' this->subproc->arenas`)
|
||||
size_t arena_idx; // index in the arenas array
|
||||
|
||||
size_t slice_count; // total size of the area in arena slices (of `MI_ARENA_SLICE_SIZE`)
|
||||
size_t info_slices; // initial slices reserved for the arena bitmaps
|
||||
int numa_node; // associated NUMA node
|
||||
bool is_exclusive; // only allow allocations if specifically for this arena
|
||||
mi_decl_align(8) // needed on some 32-bit platforms
|
||||
_Atomic(mi_msecs_t) purge_expire; // expiration time when slices can be purged from `slices_purge`.
|
||||
mi_commit_fun_t* commit_fun; // custom commit/decommit memory
|
||||
void* commit_fun_arg; // user argument for a custom commit function
|
||||
|
||||
size_t total_size; // for (user given) memory more than MI_ARENA_MAX_SIZE, we use N arena's to cover it. The first (parent) has the total size (and the other sub-arena's 0).
|
||||
mi_arena_t* parent; // if this is a sub arena, this points to the first one in the memory area.
|
||||
|
||||
mi_bbitmap_t* slices_free; // is the slice free? (a binned bitmap with size classes)
|
||||
mi_bitmap_t* slices_committed; // is the slice committed? (i.e. accessible)
|
||||
mi_bitmap_t* slices_dirty; // is the slice potentially non-zero?
|
||||
mi_bitmap_t* slices_purge; // slices that can be purged
|
||||
mi_page_t* pages_meta; // pre-allocated `slice_count` page meta info -- only used if `MI_PAGE_META_IS_SEPARATED!=0`
|
||||
mi_arena_pages_t pages_main; // arena page bitmaps for the main heap are allocated up front as well
|
||||
|
||||
// followed by the bitmaps (whose sizes depend on the arena size)
|
||||
// note: when adding bitmaps revise `mi_arena_info_slices_needed`
|
||||
} mi_arena_t;
|
||||
|
||||
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Error codes passed to `_mi_fatal_error`
|
||||
All are recoverable but EFAULT is a serious error and aborts by default in secure mode.
|
||||
For portability define undefined error codes using common Unix codes:
|
||||
<https://www-numi.fnal.gov/offline_software/srt_public_context/WebDocs/Errors/unix_system_errors.html>
|
||||
----------------------------------------------------------- */
|
||||
|
||||
#ifndef EAGAIN // double free
|
||||
#define EAGAIN (11)
|
||||
#endif
|
||||
#ifndef ENOMEM // out of memory
|
||||
#define ENOMEM (12)
|
||||
#endif
|
||||
#ifndef EFAULT // corrupted free-list or meta-data
|
||||
#define EFAULT (14)
|
||||
#endif
|
||||
#ifndef EINVAL // trying to free an invalid pointer
|
||||
#define EINVAL (22)
|
||||
#endif
|
||||
#ifndef EOVERFLOW // count*size overflow
|
||||
#define EOVERFLOW (75)
|
||||
#endif
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Debug constants
|
||||
----------------------------------------------------------- */
|
||||
|
||||
#if !defined(MI_DEBUG_UNINIT)
|
||||
#define MI_DEBUG_UNINIT (0xD0)
|
||||
#endif
|
||||
#if !defined(MI_DEBUG_FREED)
|
||||
#define MI_DEBUG_FREED (0xDF)
|
||||
#endif
|
||||
#if !defined(MI_DEBUG_PADDING)
|
||||
#define MI_DEBUG_PADDING (0xDE)
|
||||
#endif
|
||||
|
||||
|
||||
#endif // MI_TYPES_H
|
||||
Reference in New Issue
Block a user