[llvm] [Support] Vendor rpmalloc in-tree and use it for the Windows 64-bit release (PR #91862)
Alexandre Ganea via llvm-commits
llvm-commits at lists.llvm.org
Sat May 18 07:54:01 PDT 2024
https://github.com/aganea updated https://github.com/llvm/llvm-project/pull/91862
>From edbbbd139aeec9fbdb8eae15c07676032816d107 Mon Sep 17 00:00:00 2001
From: Alexandre Ganea <alex_toresh at yahoo.fr>
Date: Sun, 29 Jan 2023 18:37:37 -0500
Subject: [PATCH 1/8] [Support] Vendor rpmalloc 1.4.4 and build the official
LLVM release with it
---
llvm/CMakeLists.txt | 19 +-
llvm/docs/CMake.rst | 10 +
llvm/docs/ReleaseNotes.rst | 3 +
llvm/lib/Support/CMakeLists.txt | 3 +-
llvm/lib/Support/rpmalloc/CACHE.md | 19 +
llvm/lib/Support/rpmalloc/README.md | 173 +
llvm/lib/Support/rpmalloc/malloc.c | 657 ++++
llvm/lib/Support/rpmalloc/rpmalloc.c | 3882 +++++++++++++++++++++
llvm/lib/Support/rpmalloc/rpmalloc.h | 422 +++
llvm/lib/Support/rpmalloc/rpnew.h | 109 +
llvm/utils/release/build_llvm_release.bat | 3 +
11 files changed, 5298 insertions(+), 2 deletions(-)
create mode 100644 llvm/lib/Support/rpmalloc/CACHE.md
create mode 100644 llvm/lib/Support/rpmalloc/README.md
create mode 100644 llvm/lib/Support/rpmalloc/malloc.c
create mode 100644 llvm/lib/Support/rpmalloc/rpmalloc.c
create mode 100644 llvm/lib/Support/rpmalloc/rpmalloc.h
create mode 100644 llvm/lib/Support/rpmalloc/rpnew.h
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index c06e661573ed4..4f20f8dbb1849 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -733,7 +733,24 @@ if( WIN32 AND NOT CYGWIN )
endif()
set(LLVM_NATIVE_TOOL_DIR "" CACHE PATH "Path to a directory containing prebuilt matching native tools (such as llvm-tblgen)")
-set(LLVM_INTEGRATED_CRT_ALLOC "" CACHE PATH "Replace the Windows CRT allocator with any of {rpmalloc|mimalloc|snmalloc}. Only works with CMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded.")
+set(LLVM_ENABLE_RPMALLOC "" CACHE BOOL "Replace the CRT allocator with rpmalloc.")
+if(LLVM_ENABLE_RPMALLOC)
+ if(NOT (CMAKE_SYSTEM_NAME MATCHES "Windows|Linux"))
+ message(FATAL_ERROR "LLVM_ENABLE_RPMALLOC is only supported on Windows and Linux.")
+ endif()
+ if(LLVM_USE_SANITIZER)
+ message(FATAL_ERROR "LLVM_ENABLE_RPMALLOC cannot be used along with LLVM_USE_SANITIZER!")
+ endif()
+ if(WIN32 AND CMAKE_BUILD_TYPE AND uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+ message(FATAL_ERROR "The Debug target isn't supported along with LLVM_ENABLE_RPMALLOC!")
+ endif()
+
+ # Override the C runtime allocator with the in-tree rpmalloc
+ set(LLVM_INTEGRATED_CRT_ALLOC "${CMAKE_CURRENT_SOURCE_DIR}/lib/Support")
+ set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded")
+endif()
+
+set(LLVM_INTEGRATED_CRT_ALLOC "${LLVM_INTEGRATED_CRT_ALLOC}" CACHE PATH "Replace the Windows CRT allocator with any of {rpmalloc|mimalloc|snmalloc}. Only works with CMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded.")
if(LLVM_INTEGRATED_CRT_ALLOC)
if(NOT WIN32)
message(FATAL_ERROR "LLVM_INTEGRATED_CRT_ALLOC is only supported on Windows.")
diff --git a/llvm/docs/CMake.rst b/llvm/docs/CMake.rst
index 41ef5f40c6cf6..0b65e06d1d7b3 100644
--- a/llvm/docs/CMake.rst
+++ b/llvm/docs/CMake.rst
@@ -712,6 +712,16 @@ enabled sub-projects. Nearly all of these variable names begin with
This flag needs to be used along with the static CRT, ie. if building the
Release target, add -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded.
+ Note that rpmalloc is also supported natively in-tree, for example:
+
+ .. code-block:: console
+
+ $ D:\llvm-project> cmake ... -DLLVM_ENABLE_RPMALLOC=ON
+
+**LLVM_ENABLE_RPMALLOC**:BOOL
+ Similar to LLVM_INTEGRATED_CRT_ALLOC, embeds the in-tree rpmalloc into the
+ host toolchain as a C runtime allocator. Requires linking with the static CRT,
+ if building the Release target, with: -DLLVM_USE_CRT_RELEASE=MT.
**LLVM_LINK_LLVM_DYLIB**:BOOL
If enabled, tools will be linked with the libLLVM shared library. Defaults
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index 59c0d4dd2376d..35910e20b3fff 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -62,6 +62,9 @@ Changes to LLVM infrastructure
Changes to building LLVM
------------------------
+* LLVM now supports rpmalloc in-tree, as a replacement C allocator for hosted
+ toolchains.
+
Changes to TableGen
-------------------
diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index 03e888958a071..a7ba8aba25950 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -103,9 +103,10 @@ if(LLVM_INTEGRATED_CRT_ALLOC)
message(FATAL_ERROR "Cannot find the path to `git clone` for the CRT allocator! (${LLVM_INTEGRATED_CRT_ALLOC}). Currently, rpmalloc, snmalloc and mimalloc are supported.")
endif()
- if(LLVM_INTEGRATED_CRT_ALLOC MATCHES "rpmalloc$")
+ if((LLVM_INTEGRATED_CRT_ALLOC MATCHES "rpmalloc$") OR LLVM_ENABLE_RPMALLOC)
add_compile_definitions(ENABLE_OVERRIDE ENABLE_PRELOAD)
set(ALLOCATOR_FILES "${LLVM_INTEGRATED_CRT_ALLOC}/rpmalloc/rpmalloc.c")
+ set(delayload_flags "${delayload_flags} -INCLUDE:malloc")
elseif(LLVM_INTEGRATED_CRT_ALLOC MATCHES "snmalloc$")
set(ALLOCATOR_FILES "${LLVM_INTEGRATED_CRT_ALLOC}/src/snmalloc/override/new.cc")
set(system_libs ${system_libs} "mincore.lib" "-INCLUDE:malloc")
diff --git a/llvm/lib/Support/rpmalloc/CACHE.md b/llvm/lib/Support/rpmalloc/CACHE.md
new file mode 100644
index 0000000000000..052320baf5327
--- /dev/null
+++ b/llvm/lib/Support/rpmalloc/CACHE.md
@@ -0,0 +1,19 @@
+# Thread caches
+rpmalloc has a thread cache of free memory blocks which can be used in allocations without interfering with other threads or going to system to map more memory, as well as a global cache shared by all threads to let spans of memory pages flow between threads. Configuring the size of these caches can be crucial to obtaining good performance while minimizing memory overhead blowup. Below is a simple case study using the benchmark tool to compare different thread cache configurations for rpmalloc.
+
+The rpmalloc thread cache is configured to be unlimited, performance oriented as meaning default values, size oriented where both thread cache and global cache is reduced significantly, or disabled where both thread and global caches are disabled and completely free pages are directly unmapped.
+
+The benchmark is configured to run threads allocating 150000 blocks distributed in the `[16, 16000]` bytes range with a linear falloff probability. It runs 1000 loops, and every iteration 75000 blocks (50%) are freed and allocated in a scattered pattern. There are no cross thread allocations/deallocations. Parameters: `benchmark n 0 0 0 1000 150000 75000 16 16000`. The benchmarks are run on an Ubuntu 16.10 machine with 8 cores (4 physical, HT) and 12GiB RAM.
+
+The benchmark also includes results for the standard library malloc implementation as a reference for comparison with the nocache setting.
+
+![Ubuntu 16.10 random [16, 16000] bytes, 8 cores](https://docs.google.com/spreadsheets/d/1NWNuar1z0uPCB5iVS_Cs6hSo2xPkTmZf0KsgWS_Fb_4/pubchart?oid=387883204&format=image)
+![Ubuntu 16.10 random [16, 16000] bytes, 8 cores](https://docs.google.com/spreadsheets/d/1NWNuar1z0uPCB5iVS_Cs6hSo2xPkTmZf0KsgWS_Fb_4/pubchart?oid=1644710241&format=image)
+
+For single threaded case the unlimited cache and performance oriented cache settings have identical performance and memory overhead, indicating that the memory pages fit in the combined thread and global cache. As number of threads increase to 2-4 threads, the performance settings have slightly higher performance which can seem odd at first, but can be explained by low contention on the global cache where some memory pages can flow between threads without stalling, reducing the overall number of calls to map new memory pages (also indicated by the slightly lower memory overhead).
+
+As threads increase even more to 5-10 threads, the increased contention and eventual limit of global cache cause the unlimited setting to gain a slight advantage in performance. As expected the memory overhead remains constant for unlimited caches, while going down for performance setting when number of threads increases.
+
+The size oriented setting maintain good performance compared to the standard library while reducing the memory overhead compared to the performance setting with a decent amount.
+
+The nocache setting still outperforms the reference standard library allocator for workloads up to 6 threads while maintaining a near zero memory overhead, which is even slightly lower than the standard library. For use case scenarios where number of allocation of each size class is lower the overhead in rpmalloc from the 64KiB span size will of course increase.
diff --git a/llvm/lib/Support/rpmalloc/README.md b/llvm/lib/Support/rpmalloc/README.md
new file mode 100644
index 0000000000000..5fba25ebc8095
--- /dev/null
+++ b/llvm/lib/Support/rpmalloc/README.md
@@ -0,0 +1,173 @@
+# rpmalloc - General Purpose Memory Allocator
+This library provides a public domain cross platform lock free thread caching 16-byte aligned memory allocator implemented in C. The latest source code is always available at https://github.com/mjansson/rpmalloc
+
+Created by Mattias Jansson ([@maniccoder](https://twitter.com/maniccoder)) - Discord server for discussions at https://discord.gg/M8BwTQrt6c
+
+Platforms currently supported:
+
+- Windows
+- MacOS
+- iOS
+- Linux
+- Android
+- Haiku
+
+The code should be easily portable to any platform with atomic operations and an mmap-style virtual memory management API. The API used to map/unmap memory pages can be configured in runtime to a custom implementation and mapping granularity/size.
+
+This library is put in the public domain; you can redistribute it and/or modify it without any restrictions. Or, if you choose, you can use it under the MIT license.
+
+# Performance
+We believe rpmalloc is faster than most popular memory allocators like tcmalloc, hoard, ptmalloc3 and others without causing extra allocated memory overhead in the thread caches compared to these allocators. We also believe the implementation to be easier to read and modify compared to these allocators, as it is a single source file of ~3000 lines of C code. All allocations have a natural 16-byte alignment.
+
+Contained in a parallel repository is a benchmark utility that performs interleaved unaligned allocations and deallocations (both in-thread and cross-thread) in multiple threads. It measures number of memory operations performed per CPU second, as well as memory overhead by comparing the virtual memory mapped with the number of bytes requested in allocation calls. The setup of number of thread, cross-thread deallocation rate and allocation size limits is configured by command line arguments.
+
+https://github.com/mjansson/rpmalloc-benchmark
+
+Below is an example performance comparison chart of rpmalloc and other popular allocator implementations, with default configurations used.
+
+![Ubuntu 16.10, random [16, 8000] bytes, 8 cores](https://docs.google.com/spreadsheets/d/1NWNuar1z0uPCB5iVS_Cs6hSo2xPkTmZf0KsgWS_Fb_4/pubchart?oid=301017877&format=image)
+
+The benchmark producing these numbers were run on an Ubuntu 16.10 machine with 8 logical cores (4 physical, HT). The actual numbers are not to be interpreted as absolute performance figures, but rather as relative comparisons between the different allocators. For additional benchmark results, see the [BENCHMARKS](BENCHMARKS.md) file.
+
+Configuration of the thread and global caches can be important depending on your use pattern. See [CACHE](CACHE.md) for a case study and some comments/guidelines.
+
+# Required functions
+
+Before calling any other function in the API, you __MUST__ call the initialization function, either __rpmalloc_initialize__ or __rpmalloc_initialize_config__, or you will get undefined behaviour when calling other rpmalloc entry point.
+
+Before terminating your use of the allocator, you __SHOULD__ call __rpmalloc_finalize__ in order to release caches and unmap virtual memory, as well as prepare the allocator for global scope cleanup at process exit or dynamic library unload depending on your use case.
+
+# Using
+The easiest way to use the library is simply adding __rpmalloc.[h|c]__ to your project and compile them along with your sources. This contains only the rpmalloc specific entry points and does not provide internal hooks to process and/or thread creation at the moment. You are required to call these functions from your own code in order to initialize and finalize the allocator in your process and threads:
+
+__rpmalloc_initialize__ : Call at process start to initialize the allocator
+
+__rpmalloc_initialize_config__ : Optional entry point to call at process start to initialize the allocator with a custom memory mapping backend, memory page size and mapping granularity.
+
+__rpmalloc_finalize__: Call at process exit to finalize the allocator
+
+__rpmalloc_thread_initialize__: Call at each thread start to initialize the thread local data for the allocator
+
+__rpmalloc_thread_finalize__: Call at each thread exit to finalize and release thread cache back to global cache
+
+__rpmalloc_config__: Get the current runtime configuration of the allocator
+
+Then simply use the __rpmalloc__/__rpfree__ and the other malloc style replacement functions. Remember all allocations are 16-byte aligned, so no need to call the explicit rpmemalign/rpaligned_alloc/rpposix_memalign functions unless you need greater alignment, they are simply wrappers to make it easier to replace in existing code.
+
+If you wish to override the standard library malloc family of functions and have automatic initialization/finalization of process and threads, define __ENABLE_OVERRIDE__ to non-zero which will include the `malloc.c` file in compilation of __rpmalloc.c__. The list of libc entry points replaced may not be complete, use libc replacement only as a convenience for testing the library on an existing code base, not a final solution.
+
+For explicit first class heaps, see the __rpmalloc_heap_*__ API under [first class heaps](#first-class-heaps) section, requiring __RPMALLOC_FIRST_CLASS_HEAPS__ tp be defined to 1.
+
+# Building
+To compile as a static library run the configure python script which generates a Ninja build script, then build using ninja. The ninja build produces two static libraries, one named `rpmalloc` and one named `rpmallocwrap`, where the latter includes the libc entry point overrides.
+
+The configure + ninja build also produces two shared object/dynamic libraries. The `rpmallocwrap` shared library can be used with LD_PRELOAD/DYLD_INSERT_LIBRARIES to inject in a preexisting binary, replacing any malloc/free family of function calls. This is only implemented for Linux and macOS targets. The list of libc entry points replaced may not be complete, use preloading as a convenience for testing the library on an existing binary, not a final solution. The dynamic library also provides automatic init/fini of process and threads for all platforms.
+
+The latest stable release is available in the master branch. For latest development code, use the develop branch.
+
+# Cache configuration options
+Free memory pages are cached both per thread and in a global cache for all threads. The size of the thread caches is determined by an adaptive scheme where each cache is limited by a percentage of the maximum allocation count of the corresponding size class. The size of the global caches is determined by a multiple of the maximum of all thread caches. The factors controlling the cache sizes can be set by editing the individual defines in the `rpmalloc.c` source file for fine tuned control.
+
+__ENABLE_UNLIMITED_CACHE__: By default defined to 0, set to 1 to make all caches infinite, i.e never release spans to global cache unless thread finishes and never unmap memory pages back to the OS. Highest performance but largest memory overhead.
+
+__ENABLE_UNLIMITED_GLOBAL_CACHE__: By default defined to 0, set to 1 to make global caches infinite, i.e never unmap memory pages back to the OS.
+
+__ENABLE_UNLIMITED_THREAD_CACHE__: By default defined to 0, set to 1 to make thread caches infinite, i.e never release spans to global cache unless thread finishes.
+
+__ENABLE_GLOBAL_CACHE__: By default defined to 1, enables the global cache shared between all threads. Set to 0 to disable the global cache and directly unmap pages evicted from the thread cache.
+
+__ENABLE_THREAD_CACHE__: By default defined to 1, enables the per-thread cache. Set to 0 to disable the thread cache and directly unmap pages no longer in use (also disables the global cache).
+
+__ENABLE_ADAPTIVE_THREAD_CACHE__: Introduces a simple heuristics in the thread cache size, keeping 25% of the high water mark for each span count class.
+
+# Other configuration options
+Detailed statistics are available if __ENABLE_STATISTICS__ is defined to 1 (default is 0, or disabled), either on compile command line or by setting the value in `rpmalloc.c`. This will cause a slight overhead in runtime to collect statistics for each memory operation, and will also add 4 bytes overhead per allocation to track sizes.
+
+Integer safety checks on all calls are enabled if __ENABLE_VALIDATE_ARGS__ is defined to 1 (default is 0, or disabled), either on compile command line or by setting the value in `rpmalloc.c`. If enabled, size arguments to the global entry points are verified not to cause integer overflows in calculations.
+
+Asserts are enabled if __ENABLE_ASSERTS__ is defined to 1 (default is 0, or disabled), either on compile command line or by setting the value in `rpmalloc.c`.
+
+To include __malloc.c__ in compilation and provide overrides of standard library malloc entry points define __ENABLE_OVERRIDE__ to 1. To enable automatic initialization of finalization of process and threads in order to preload the library into executables using standard library malloc, define __ENABLE_PRELOAD__ to 1.
+
+To enable the runtime configurable memory page and span sizes, define __RPMALLOC_CONFIGURABLE__ to 1. By default, memory page size is determined by system APIs and memory span size is set to 64KiB.
+
+To enable support for first class heaps, define __RPMALLOC_FIRST_CLASS_HEAPS__ to 1. By default, the first class heap API is disabled.
+
+# Huge pages
+The allocator has support for huge/large pages on Windows, Linux and MacOS. To enable it, pass a non-zero value in the config value `enable_huge_pages` when initializing the allocator with `rpmalloc_initialize_config`. If the system does not support huge pages it will be automatically disabled. You can query the status by looking at `enable_huge_pages` in the config returned from a call to `rpmalloc_config` after initialization is done.
+
+# Quick overview
+The allocator is similar in spirit to tcmalloc from the [Google Performance Toolkit](https://github.com/gperftools/gperftools). It uses separate heaps for each thread and partitions memory blocks according to a preconfigured set of size classes, up to 2MiB. Larger blocks are mapped and unmapped directly. Allocations for different size classes will be served from different set of memory pages, each "span" of pages is dedicated to one size class. Spans of pages can flow between threads when the thread cache overflows and are released to a global cache, or when the thread ends. Unlike tcmalloc, single blocks do not flow between threads, only entire spans of pages.
+
+# Implementation details
+The allocator is based on a fixed but configurable page alignment (defaults to 64KiB) and 16 byte block alignment, where all runs of memory pages (spans) are mapped to this alignment boundary. On Windows this is automatically guaranteed up to 64KiB by the VirtualAlloc granularity, and on mmap systems it is achieved by oversizing the mapping and aligning the returned virtual memory address to the required boundaries. By aligning to a fixed size the free operation can locate the header of the memory span without having to do a table lookup (as tcmalloc does) by simply masking out the low bits of the address (for 64KiB this would be the low 16 bits).
+
+Memory blocks are divided into three categories. For 64KiB span size/alignment the small blocks are [16, 1024] bytes, medium blocks (1024, 32256] bytes, and large blocks (32256, 2097120] bytes. The three categories are further divided in size classes. If the span size is changed, the small block classes remain but medium blocks go from (1024, span size] bytes.
+
+Small blocks have a size class granularity of 16 bytes each in 64 buckets. Medium blocks have a granularity of 512 bytes, 61 buckets (default). Large blocks have the same granularity as the configured span size (default 64KiB). All allocations are fitted to these size class boundaries (an allocation of 36 bytes will allocate a block of 48 bytes). Each small and medium size class has an associated span (meaning a contiguous set of memory pages) configuration describing how many pages the size class will allocate each time the cache is empty and a new allocation is requested.
+
+Spans for small and medium blocks are cached in four levels to avoid calls to map/unmap memory pages. The first level is a per thread single active span for each size class. The second level is a per thread list of partially free spans for each size class. The third level is a per thread list of free spans. The fourth level is a global list of free spans.
+
+Each span for a small and medium size class keeps track of how many blocks are allocated/free, as well as a list of which blocks that are free for allocation. To avoid locks, each span is completely owned by the allocating thread, and all cross-thread deallocations will be deferred to the owner thread through a separate free list per span.
+
+Large blocks, or super spans, are cached in two levels. The first level is a per thread list of free super spans. The second level is a global list of free super spans.
+
+# Memory mapping
+By default the allocator uses OS APIs to map virtual memory pages as needed, either `VirtualAlloc` on Windows or `mmap` on POSIX systems. If you want to use your own custom memory mapping provider you can use __rpmalloc_initialize_config__ and pass function pointers to map and unmap virtual memory. These function should reserve and free the requested number of bytes.
+
+The returned memory address from the memory map function MUST be aligned to the memory page size and the memory span size (which ever is larger), both of which is configurable. Either provide the page and span sizes during initialization using __rpmalloc_initialize_config__, or use __rpmalloc_config__ to find the required alignment which is equal to the maximum of page and span size. The span size MUST be a power of two in [4096, 262144] range, and be a multiple or divisor of the memory page size.
+
+Memory mapping requests are always done in multiples of the memory page size. You can specify a custom page size when initializing rpmalloc with __rpmalloc_initialize_config__, or pass 0 to let rpmalloc determine the system memory page size using OS APIs. The page size MUST be a power of two.
+
+To reduce system call overhead, memory spans are mapped in batches controlled by the `span_map_count` configuration variable (which defaults to the `DEFAULT_SPAN_MAP_COUNT` value if 0, which in turn is sized according to the cache configuration define, defaulting to 64). If the memory page size is larger than the span size, the number of spans to map in a single call will be adjusted to guarantee a multiple of the page size, and the spans will be kept mapped until the entire span range can be unmapped in one call (to avoid trying to unmap partial pages).
+
+On macOS and iOS mmap requests are tagged with tag 240 for easy identification with the vmmap tool.
+
+# Span breaking
+Super spans (spans a multiple > 1 of the span size) can be subdivided into smaller spans to fulfill a need to map a new span of memory. By default the allocator will greedily grab and break any larger span from the available caches before mapping new virtual memory. However, spans can currently not be glued together to form larger super spans again. Subspans can traverse the cache and be used by different threads individually.
+
+A span that is a subspan of a larger super span can be individually decommitted to reduce physical memory pressure when the span is evicted from caches and scheduled to be unmapped. The entire original super span will keep track of the subspans it is broken up into, and when the entire range is decommitted the super span will be unmapped. This allows platforms like Windows that require the entire virtual memory range that was mapped in a call to VirtualAlloc to be unmapped in one call to VirtualFree, while still decommitting individual pages in subspans (if the page size is smaller than the span size).
+
+If you use a custom memory map/unmap function you need to take this into account by looking at the `release` parameter given to the `memory_unmap` function. It is set to 0 for decommitting individual pages and the total super span byte size for finally releasing the entire super span memory range.
+
+# Memory fragmentation
+There is no memory fragmentation by the allocator in the sense that it will not leave unallocated and unusable "holes" in the memory pages by calls to allocate and free blocks of different sizes. This is due to the fact that the memory pages allocated for each size class is split up in perfectly aligned blocks which are not reused for a request of a different size. The block freed by a call to `rpfree` will always be immediately available for an allocation request within the same size class.
+
+However, there is memory fragmentation in the meaning that a request for x bytes followed by a request of y bytes where x and y are at least one size class different in size will return blocks that are at least one memory page apart in virtual address space. Only blocks of the same size will potentially be within the same memory page span.
+
+rpmalloc keeps an "active span" and free list for each size class. This leads to back-to-back allocations will most likely be served from within the same span of memory pages (unless the span runs out of free blocks). The rpmalloc implementation will also use any "holes" in memory pages in semi-filled spans before using a completely free span.
+
+# First class heaps
+rpmalloc provides a first class heap type with explicit heap control API. Heaps are maintained with calls to __rpmalloc_heap_acquire__ and __rpmalloc_heap_release__ and allocations/frees are done with __rpmalloc_heap_alloc__ and __rpmalloc_heap_free__. See the `rpmalloc.h` documentation for the full list of functions in the heap API. The main use case of explicit heap control is to scope allocations in a heap and release everything with a single call to __rpmalloc_heap_free_all__ without having to maintain ownership of memory blocks. Note that the heap API is not thread-safe, the caller must make sure that each heap is only used in a single thread at any given time.
+
+# Producer-consumer scenario
+Compared to the some other allocators, rpmalloc does not suffer as much from a producer-consumer thread scenario where one thread allocates memory blocks and another thread frees the blocks. In some allocators the free blocks need to traverse both the thread cache of the thread doing the free operations as well as the global cache before being reused in the allocating thread. In rpmalloc the freed blocks will be reused as soon as the allocating thread needs to get new spans from the thread cache. This enables faster release of completely freed memory pages as blocks in a memory page will not be aliased between different owning threads.
+
+# Best case scenarios
+Threads that keep ownership of allocated memory blocks within the thread and free the blocks from the same thread will have optimal performance.
+
+Threads that have allocation patterns where the difference in memory usage high and low water marks fit within the thread cache thresholds in the allocator will never touch the global cache except during thread init/fini and have optimal performance. Tweaking the cache limits can be done on a per-size-class basis.
+
+# Worst case scenarios
+Since each thread cache maps spans of memory pages per size class, a thread that allocates just a few blocks of each size class (16, 32, ...) for many size classes will never fill each bucket, and thus map a lot of memory pages while only using a small fraction of the mapped memory. However, the wasted memory will always be less than 4KiB (or the configured memory page size) per size class as each span is initialized one memory page at a time. The cache for free spans will be reused by all size classes.
+
+Threads that perform a lot of allocations and deallocations in a pattern that have a large difference in high and low water marks, and that difference is larger than the thread cache size, will put a lot of contention on the global cache. What will happen is the thread cache will overflow on each low water mark causing pages to be released to the global cache, then underflow on high water mark causing pages to be re-acquired from the global cache. This can be mitigated by changing the __MAX_SPAN_CACHE_DIVISOR__ define in the source code (at the cost of higher average memory overhead).
+
+# Caveats
+VirtualAlloc has an internal granularity of 64KiB. However, mmap lacks this granularity control, and the implementation instead oversizes the memory mapping with configured span size to be able to always return a memory area with the required alignment. Since the extra memory pages are never touched this will not result in extra committed physical memory pages, but rather only increase virtual memory address space.
+
+All entry points assume the passed values are valid, for example passing an invalid pointer to free would most likely result in a segmentation fault. __The library does not try to guard against errors!__.
+
+To support global scope data doing dynamic allocation/deallocation such as C++ objects with custom constructors and destructors, the call to __rpmalloc_finalize__ will not completely terminate the allocator but rather empty all caches and put the allocator in finalization mode. Once this call has been made, the allocator is no longer thread safe and expects all remaining calls to originate from global data destruction on main thread. Any spans or heaps becoming free during this phase will be immediately unmapped to allow correct teardown of the process or dynamic library without any leaks.
+
+# Other languages
+
+[Johan Andersson](https://github.com/repi) at Embark has created a Rust wrapper available at [rpmalloc-rs](https://github.com/EmbarkStudios/rpmalloc-rs)
+
+[Stas Denisov](https://github.com/nxrighthere) has created a C# wrapper available at [Rpmalloc-CSharp](https://github.com/nxrighthere/Rpmalloc-CSharp)
+
+# License
+
+As part of the LLVM codebase, rpmalloc has be re-licenced under the the Apache License v2.0 with LLVM Exceptions.
+See https://llvm.org/LICENSE.txt for license information.
+SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
diff --git a/llvm/lib/Support/rpmalloc/malloc.c b/llvm/lib/Support/rpmalloc/malloc.c
new file mode 100644
index 0000000000000..d4f99e5fe1d04
--- /dev/null
+++ b/llvm/lib/Support/rpmalloc/malloc.c
@@ -0,0 +1,657 @@
+//===------------------------ malloc.c -----------------*- C -*-=============//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This library provides a cross-platform lock free thread caching malloc
+// implementation in C11.
+//
+// This file provides overrides for the standard library malloc entry points for
+// C and new/delete operators for C++. It also provides automatic
+// initialization/finalization of process and threads
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(__TINYC__)
+#include <sys/types.h>
+#endif
+
+#ifndef ARCH_64BIT
+#if defined(__LLP64__) || defined(__LP64__) || defined(_WIN64)
+#define ARCH_64BIT 1
+_Static_assert(sizeof(size_t) == 8, "Data type size mismatch");
+_Static_assert(sizeof(void *) == 8, "Data type size mismatch");
+#else
+#define ARCH_64BIT 0
+_Static_assert(sizeof(size_t) == 4, "Data type size mismatch");
+_Static_assert(sizeof(void *) == 4, "Data type size mismatch");
+#endif
+#endif
+
+#if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__)
+#pragma GCC visibility push(default)
+#endif
+
+#define USE_IMPLEMENT 1
+#define USE_INTERPOSE 0
+#define USE_ALIAS 0
+
+#if defined(__APPLE__) && ENABLE_PRELOAD
+#undef USE_INTERPOSE
+#define USE_INTERPOSE 1
+
+typedef struct interpose_t {
+ void *new_func;
+ void *orig_func;
+} interpose_t;
+
+#define MAC_INTERPOSE_PAIR(newf, oldf) \
+ { (void *)newf, (void *)oldf }
+#define MAC_INTERPOSE_SINGLE(newf, oldf) \
+ __attribute__((used)) static const interpose_t macinterpose##newf##oldf \
+ __attribute__((section("__DATA, __interpose"))) = \
+ MAC_INTERPOSE_PAIR(newf, oldf)
+
+#endif
+
+#if !defined(_WIN32) && !USE_INTERPOSE
+#undef USE_IMPLEMENT
+#undef USE_ALIAS
+#define USE_IMPLEMENT 0
+#define USE_ALIAS 1
+#endif
+
+#ifdef _MSC_VER
+#pragma warning(disable : 4100)
+#undef malloc
+#undef free
+#undef calloc
+#define RPMALLOC_RESTRICT __declspec(restrict)
+#else
+#define RPMALLOC_RESTRICT
+#endif
+
+#if ENABLE_OVERRIDE
+
+typedef struct rp_nothrow_t {
+ int __dummy;
+} rp_nothrow_t;
+
+#if USE_IMPLEMENT
+
+extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL malloc(size_t size) {
+ return rpmalloc(size);
+}
+extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL calloc(size_t count,
+ size_t size) {
+ return rpcalloc(count, size);
+}
+extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL realloc(void *ptr,
+ size_t size) {
+ return rprealloc(ptr, size);
+}
+extern inline void *RPMALLOC_CDECL reallocf(void *ptr, size_t size) {
+ return rprealloc(ptr, size);
+}
+extern inline void *RPMALLOC_CDECL aligned_alloc(size_t alignment,
+ size_t size) {
+ return rpaligned_alloc(alignment, size);
+}
+extern inline void *RPMALLOC_CDECL memalign(size_t alignment, size_t size) {
+ return rpmemalign(alignment, size);
+}
+extern inline int RPMALLOC_CDECL posix_memalign(void **memptr, size_t alignment,
+ size_t size) {
+ return rpposix_memalign(memptr, alignment, size);
+}
+extern inline void RPMALLOC_CDECL free(void *ptr) { rpfree(ptr); }
+extern inline void RPMALLOC_CDECL cfree(void *ptr) { rpfree(ptr); }
+extern inline size_t RPMALLOC_CDECL malloc_usable_size(void *ptr) {
+ return rpmalloc_usable_size(ptr);
+}
+extern inline size_t RPMALLOC_CDECL malloc_size(void *ptr) {
+ return rpmalloc_usable_size(ptr);
+}
+
+#ifdef _WIN32
+// For Windows, #include <rpnew.h> in one source file to get the C++ operator
+// overrides implemented in your module
+#else
+// Overload the C++ operators using the mangled names
+// (https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling) operators
+// delete and delete[]
+extern void _ZdlPv(void *p);
+void _ZdlPv(void *p) { rpfree(p); }
+extern void _ZdaPv(void *p);
+void _ZdaPv(void *p) { rpfree(p); }
+#if ARCH_64BIT
+// 64-bit operators new and new[], normal and aligned
+extern void *_Znwm(uint64_t size);
+void *_Znwm(uint64_t size) { return rpmalloc(size); }
+extern void *_Znam(uint64_t size);
+void *_Znam(uint64_t size) { return rpmalloc(size); }
+extern void *_Znwmm(uint64_t size, uint64_t align);
+void *_Znwmm(uint64_t size, uint64_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern void *_Znamm(uint64_t size, uint64_t align);
+void *_Znamm(uint64_t size, uint64_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern void *_ZnwmSt11align_val_t(uint64_t size, uint64_t align);
+void *_ZnwmSt11align_val_t(uint64_t size, uint64_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern void *_ZnamSt11align_val_t(uint64_t size, uint64_t align);
+void *_ZnamSt11align_val_t(uint64_t size, uint64_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern void *_ZnwmRKSt9nothrow_t(uint64_t size, rp_nothrow_t t);
+void *_ZnwmRKSt9nothrow_t(uint64_t size, rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpmalloc(size);
+}
+extern void *_ZnamRKSt9nothrow_t(uint64_t size, rp_nothrow_t t);
+void *_ZnamRKSt9nothrow_t(uint64_t size, rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpmalloc(size);
+}
+extern void *_ZnwmSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
+ rp_nothrow_t t);
+void *_ZnwmSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
+ rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpaligned_alloc(align, size);
+}
+extern void *_ZnamSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
+ rp_nothrow_t t);
+void *_ZnamSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
+ rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpaligned_alloc(align, size);
+}
+// 64-bit operators sized delete and delete[], normal and aligned
+extern void _ZdlPvm(void *p, uint64_t size);
+void _ZdlPvm(void *p, uint64_t size) {
+ rpfree(p);
+ (void)sizeof(size);
+}
+extern void _ZdaPvm(void *p, uint64_t size);
+void _ZdaPvm(void *p, uint64_t size) {
+ rpfree(p);
+ (void)sizeof(size);
+}
+extern void _ZdlPvSt11align_val_t(void *p, uint64_t align);
+void _ZdlPvSt11align_val_t(void *p, uint64_t align) {
+ rpfree(p);
+ (void)sizeof(align);
+}
+extern void _ZdaPvSt11align_val_t(void *p, uint64_t align);
+void _ZdaPvSt11align_val_t(void *p, uint64_t align) {
+ rpfree(p);
+ (void)sizeof(align);
+}
+extern void _ZdlPvmSt11align_val_t(void *p, uint64_t size, uint64_t align);
+void _ZdlPvmSt11align_val_t(void *p, uint64_t size, uint64_t align) {
+ rpfree(p);
+ (void)sizeof(size);
+ (void)sizeof(align);
+}
+extern void _ZdaPvmSt11align_val_t(void *p, uint64_t size, uint64_t align);
+void _ZdaPvmSt11align_val_t(void *p, uint64_t size, uint64_t align) {
+ rpfree(p);
+ (void)sizeof(size);
+ (void)sizeof(align);
+}
+#else
+// 32-bit operators new and new[], normal and aligned
+extern void *_Znwj(uint32_t size);
+void *_Znwj(uint32_t size) { return rpmalloc(size); }
+extern void *_Znaj(uint32_t size);
+void *_Znaj(uint32_t size) { return rpmalloc(size); }
+extern void *_Znwjj(uint32_t size, uint32_t align);
+void *_Znwjj(uint32_t size, uint32_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern void *_Znajj(uint32_t size, uint32_t align);
+void *_Znajj(uint32_t size, uint32_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern void *_ZnwjSt11align_val_t(size_t size, size_t align);
+void *_ZnwjSt11align_val_t(size_t size, size_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern void *_ZnajSt11align_val_t(size_t size, size_t align);
+void *_ZnajSt11align_val_t(size_t size, size_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern void *_ZnwjRKSt9nothrow_t(size_t size, rp_nothrow_t t);
+void *_ZnwjRKSt9nothrow_t(size_t size, rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpmalloc(size);
+}
+extern void *_ZnajRKSt9nothrow_t(size_t size, rp_nothrow_t t);
+void *_ZnajRKSt9nothrow_t(size_t size, rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpmalloc(size);
+}
+extern void *_ZnwjSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
+ rp_nothrow_t t);
+void *_ZnwjSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
+ rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpaligned_alloc(align, size);
+}
+extern void *_ZnajSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
+ rp_nothrow_t t);
+void *_ZnajSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
+ rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpaligned_alloc(align, size);
+}
+// 32-bit operators sized delete and delete[], normal and aligned
+extern void _ZdlPvj(void *p, uint64_t size);
+void _ZdlPvj(void *p, uint64_t size) {
+ rpfree(p);
+ (void)sizeof(size);
+}
+extern void _ZdaPvj(void *p, uint64_t size);
+void _ZdaPvj(void *p, uint64_t size) {
+ rpfree(p);
+ (void)sizeof(size);
+}
+extern void _ZdlPvSt11align_val_t(void *p, uint32_t align);
+void _ZdlPvSt11align_val_t(void *p, uint64_t a) {
+ rpfree(p);
+ (void)sizeof(align);
+}
+extern void _ZdaPvSt11align_val_t(void *p, uint32_t align);
+void _ZdaPvSt11align_val_t(void *p, uint64_t a) {
+ rpfree(p);
+ (void)sizeof(align);
+}
+extern void _ZdlPvjSt11align_val_t(void *p, uint32_t size, uint32_t align);
+void _ZdlPvjSt11align_val_t(void *p, uint64_t size, uint64_t align) {
+ rpfree(p);
+ (void)sizeof(size);
+ (void)sizeof(a);
+}
+extern void _ZdaPvjSt11align_val_t(void *p, uint32_t size, uint32_t align);
+void _ZdaPvjSt11align_val_t(void *p, uint64_t size, uint64_t align) {
+ rpfree(p);
+ (void)sizeof(size);
+ (void)sizeof(a);
+}
+#endif
+#endif
+
+#endif
+
+#if USE_INTERPOSE
+
+__attribute__((used)) static const interpose_t macinterpose_malloc[]
+ __attribute__((section("__DATA, __interpose"))) = {
+ // new and new[]
+ MAC_INTERPOSE_PAIR(rpmalloc, _Znwm),
+ MAC_INTERPOSE_PAIR(rpmalloc, _Znam),
+ // delete and delete[]
+ MAC_INTERPOSE_PAIR(rpfree, _ZdlPv), MAC_INTERPOSE_PAIR(rpfree, _ZdaPv),
+ MAC_INTERPOSE_PAIR(rpmalloc, malloc),
+ MAC_INTERPOSE_PAIR(rpmalloc, calloc),
+ MAC_INTERPOSE_PAIR(rprealloc, realloc),
+ MAC_INTERPOSE_PAIR(rprealloc, reallocf),
+#if defined(__MAC_10_15) && __MAC_OS_X_VERSION_MIN_REQUIRED >= __MAC_10_15
+ MAC_INTERPOSE_PAIR(rpaligned_alloc, aligned_alloc),
+#endif
+ MAC_INTERPOSE_PAIR(rpmemalign, memalign),
+ MAC_INTERPOSE_PAIR(rpposix_memalign, posix_memalign),
+ MAC_INTERPOSE_PAIR(rpfree, free), MAC_INTERPOSE_PAIR(rpfree, cfree),
+ MAC_INTERPOSE_PAIR(rpmalloc_usable_size, malloc_usable_size),
+ MAC_INTERPOSE_PAIR(rpmalloc_usable_size, malloc_size)};
+
+#endif
+
+#if USE_ALIAS
+
+#define RPALIAS(fn) __attribute__((alias(#fn), used, visibility("default")));
+
+// Alias the C++ operators using the mangled names
+// (https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling)
+
+// operators delete and delete[]
+void _ZdlPv(void *p) RPALIAS(rpfree) void _ZdaPv(void *p)
+ RPALIAS(rpfree) extern inline void _ZdlPvm(void *p, size_t n) {
+ rpfree(p);
+ (void)sizeof(n);
+}
+extern inline void _ZdaPvm(void *p, size_t n) {
+ rpfree(p);
+ (void)sizeof(n);
+}
+extern inline void _ZdlPvSt11align_val_t(void *p, size_t a) {
+ rpfree(p);
+ (void)sizeof(a);
+}
+extern inline void _ZdaPvSt11align_val_t(void *p, size_t a) {
+ rpfree(p);
+ (void)sizeof(a);
+}
+extern inline void _ZdlPvmSt11align_val_t(void *p, size_t n, size_t a) {
+ rpfree(p);
+ (void)sizeof(n);
+ (void)sizeof(a);
+}
+extern inline void _ZdaPvmSt11align_val_t(void *p, size_t n, size_t a) {
+ rpfree(p);
+ (void)sizeof(n);
+ (void)sizeof(a);
+}
+
+#if ARCH_64BIT
+// 64-bit operators new and new[], normal and aligned
+void *_Znwm(uint64_t size) RPALIAS(rpmalloc) void *_Znam(uint64_t size)
+ RPALIAS(rpmalloc) extern inline void *_Znwmm(uint64_t size,
+ uint64_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern inline void *_Znamm(uint64_t size, uint64_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern inline void *_ZnwmSt11align_val_t(size_t size, size_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern inline void *_ZnamSt11align_val_t(size_t size, size_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern inline void *_ZnwmRKSt9nothrow_t(size_t size, rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpmalloc(size);
+}
+extern inline void *_ZnamRKSt9nothrow_t(size_t size, rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpmalloc(size);
+}
+extern inline void *
+_ZnwmSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpaligned_alloc(align, size);
+}
+extern inline void *
+_ZnamSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpaligned_alloc(align, size);
+}
+#else
+// 32-bit operators new and new[], normal and aligned
+void *_Znwj(uint32_t size) RPALIAS(rpmalloc) void *_Znaj(uint32_t size)
+ RPALIAS(rpmalloc) extern inline void *_Znwjj(uint32_t size,
+ uint32_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern inline void *_Znajj(uint32_t size, uint32_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern inline void *_ZnwjSt11align_val_t(size_t size, size_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern inline void *_ZnajSt11align_val_t(size_t size, size_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern inline void *_ZnwjRKSt9nothrow_t(size_t size, rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpmalloc(size);
+}
+extern inline void *_ZnajRKSt9nothrow_t(size_t size, rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpmalloc(size);
+}
+extern inline void *
+_ZnwjSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpaligned_alloc(align, size);
+}
+extern inline void *
+_ZnajSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpaligned_alloc(align, size);
+}
+#endif
+
+void *malloc(size_t size)
+ RPALIAS(rpmalloc) void *calloc(size_t count, size_t size)
+ RPALIAS(rpcalloc) void *realloc(void *ptr, size_t size)
+ RPALIAS(rprealloc) void *reallocf(void *ptr, size_t size)
+ RPALIAS(rprealloc) void *aligned_alloc(size_t alignment,
+ size_t size)
+ RPALIAS(rpaligned_alloc) void *memalign(size_t alignment,
+ size_t size)
+ RPALIAS(rpmemalign) int posix_memalign(void **memptr,
+ size_t alignment,
+ size_t size)
+ RPALIAS(rpposix_memalign) void free(void *ptr)
+ RPALIAS(rpfree) void cfree(void *ptr)
+ RPALIAS(rpfree)
+#if defined(__ANDROID__) || defined(__FreeBSD__)
+ size_t
+ malloc_usable_size(const void *ptr) RPALIAS(rpmalloc_usable_size)
+#else
+ size_t
+ malloc_usable_size(void *ptr) RPALIAS(rpmalloc_usable_size)
+#endif
+ size_t malloc_size(void *ptr) RPALIAS(rpmalloc_usable_size)
+
+#endif
+
+ static inline size_t _rpmalloc_page_size(void) {
+ return _memory_page_size;
+}
+
+extern inline void *RPMALLOC_CDECL reallocarray(void *ptr, size_t count,
+ size_t size) {
+ size_t total;
+#if ENABLE_VALIDATE_ARGS
+#ifdef _MSC_VER
+ int err = SizeTMult(count, size, &total);
+ if ((err != S_OK) || (total >= MAX_ALLOC_SIZE)) {
+ errno = EINVAL;
+ return 0;
+ }
+#else
+ int err = __builtin_umull_overflow(count, size, &total);
+ if (err || (total >= MAX_ALLOC_SIZE)) {
+ errno = EINVAL;
+ return 0;
+ }
+#endif
+#else
+ total = count * size;
+#endif
+ return realloc(ptr, total);
+}
+
+extern inline void *RPMALLOC_CDECL valloc(size_t size) {
+ get_thread_heap();
+ return rpaligned_alloc(_rpmalloc_page_size(), size);
+}
+
+extern inline void *RPMALLOC_CDECL pvalloc(size_t size) {
+ get_thread_heap();
+ const size_t page_size = _rpmalloc_page_size();
+ const size_t aligned_size = ((size + page_size - 1) / page_size) * page_size;
+#if ENABLE_VALIDATE_ARGS
+ if (aligned_size < size) {
+ errno = EINVAL;
+ return 0;
+ }
+#endif
+ return rpaligned_alloc(_rpmalloc_page_size(), aligned_size);
+}
+
+#endif // ENABLE_OVERRIDE
+
+#if ENABLE_PRELOAD
+
+#ifdef _WIN32
+
+#if defined(BUILD_DYNAMIC_LINK) && BUILD_DYNAMIC_LINK
+
+extern __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE instance,
+ DWORD reason, LPVOID reserved);
+
+extern __declspec(dllexport) BOOL WINAPI
+DllMain(HINSTANCE instance, DWORD reason, LPVOID reserved) {
+ (void)sizeof(reserved);
+ (void)sizeof(instance);
+ if (reason == DLL_PROCESS_ATTACH)
+ rpmalloc_initialize();
+ else if (reason == DLL_PROCESS_DETACH)
+ rpmalloc_finalize();
+ else if (reason == DLL_THREAD_ATTACH)
+ rpmalloc_thread_initialize();
+ else if (reason == DLL_THREAD_DETACH)
+ rpmalloc_thread_finalize(1);
+ return TRUE;
+}
+
+// end BUILD_DYNAMIC_LINK
+#else
+
+extern void _global_rpmalloc_init(void) {
+ rpmalloc_set_main_thread();
+ rpmalloc_initialize();
+}
+
+#if defined(__clang__) || defined(__GNUC__)
+
+static void __attribute__((constructor)) initializer(void) {
+ _global_rpmalloc_init();
+}
+
+#elif defined(_MSC_VER)
+
+#pragma section(".CRT$XIB", read)
+__declspec(allocate(".CRT$XIB")) void (*_rpmalloc_module_init)(void) =
+ _global_rpmalloc_init;
+#pragma comment(linker, "/include:_rpmalloc_module_init")
+
+#endif
+
+// end !BUILD_DYNAMIC_LINK
+#endif
+
+#else
+
+#include <pthread.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+extern void rpmalloc_set_main_thread(void);
+
+static pthread_key_t destructor_key;
+
+static void thread_destructor(void *);
+
+static void __attribute__((constructor)) initializer(void) {
+ rpmalloc_set_main_thread();
+ rpmalloc_initialize();
+ pthread_key_create(&destructor_key, thread_destructor);
+}
+
+static void __attribute__((destructor)) finalizer(void) { rpmalloc_finalize(); }
+
+typedef struct {
+ void *(*real_start)(void *);
+ void *real_arg;
+} thread_starter_arg;
+
+static void *thread_starter(void *argptr) {
+ thread_starter_arg *arg = argptr;
+ void *(*real_start)(void *) = arg->real_start;
+ void *real_arg = arg->real_arg;
+ rpmalloc_thread_initialize();
+ rpfree(argptr);
+ pthread_setspecific(destructor_key, (void *)1);
+ return (*real_start)(real_arg);
+}
+
+static void thread_destructor(void *value) {
+ (void)sizeof(value);
+ rpmalloc_thread_finalize(1);
+}
+
+#ifdef __APPLE__
+
+static int pthread_create_proxy(pthread_t *thread, const pthread_attr_t *attr,
+ void *(*start_routine)(void *), void *arg) {
+ rpmalloc_initialize();
+ thread_starter_arg *starter_arg = rpmalloc(sizeof(thread_starter_arg));
+ starter_arg->real_start = start_routine;
+ starter_arg->real_arg = arg;
+ return pthread_create(thread, attr, thread_starter, starter_arg);
+}
+
+MAC_INTERPOSE_SINGLE(pthread_create_proxy, pthread_create);
+
+#else
+
+#include <dlfcn.h>
+
+int pthread_create(pthread_t *thread, const pthread_attr_t *attr,
+ void *(*start_routine)(void *), void *arg) {
+#if defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__) || \
+ defined(__NetBSD__) || defined(__DragonFly__) || defined(__APPLE__) || \
+ defined(__HAIKU__)
+ char fname[] = "pthread_create";
+#else
+ char fname[] = "_pthread_create";
+#endif
+ void *real_pthread_create = dlsym(RTLD_NEXT, fname);
+ rpmalloc_thread_initialize();
+ thread_starter_arg *starter_arg = rpmalloc(sizeof(thread_starter_arg));
+ starter_arg->real_start = start_routine;
+ starter_arg->real_arg = arg;
+ return (*(int (*)(pthread_t *, const pthread_attr_t *, void *(*)(void *),
+ void *))real_pthread_create)(thread, attr, thread_starter,
+ starter_arg);
+}
+
+#endif
+
+#endif
+
+#endif
+
+#if ENABLE_OVERRIDE
+
+#if defined(__GLIBC__) && defined(__linux__)
+
+void *__libc_malloc(size_t size)
+ RPALIAS(rpmalloc) void *__libc_calloc(size_t count, size_t size)
+ RPALIAS(rpcalloc) void *__libc_realloc(void *p, size_t size)
+ RPALIAS(rprealloc) void __libc_free(void *p)
+ RPALIAS(rpfree) void __libc_cfree(void *p)
+ RPALIAS(rpfree) void *__libc_memalign(size_t align,
+ size_t size)
+ RPALIAS(rpmemalign) int __posix_memalign(void **p,
+ size_t align,
+ size_t size)
+ RPALIAS(rpposix_memalign)
+
+ extern void *__libc_valloc(size_t size);
+extern void *__libc_pvalloc(size_t size);
+
+void *__libc_valloc(size_t size) { return valloc(size); }
+
+void *__libc_pvalloc(size_t size) { return pvalloc(size); }
+
+#endif
+
+#endif
+
+#if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__)
+#pragma GCC visibility pop
+#endif
diff --git a/llvm/lib/Support/rpmalloc/rpmalloc.c b/llvm/lib/Support/rpmalloc/rpmalloc.c
new file mode 100644
index 0000000000000..0f9257563a104
--- /dev/null
+++ b/llvm/lib/Support/rpmalloc/rpmalloc.c
@@ -0,0 +1,3882 @@
+//===-------------------------- rpmalloc.c ----------------*- C -*-===========//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This library provides a cross-platform lock free thread caching malloc
+// implementation in C11.
+//
+//===----------------------------------------------------------------------===//
+
+#include "rpmalloc.h"
+
+////////////
+///
+/// Build time configurable limits
+///
+//////
+
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wunused-macros"
+#pragma clang diagnostic ignored "-Wunused-function"
+#if __has_warning("-Wreserved-identifier")
+#pragma clang diagnostic ignored "-Wreserved-identifier"
+#endif
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wunused-macros"
+#pragma GCC diagnostic ignored "-Wunused-function"
+#endif
+
+#ifndef HEAP_ARRAY_SIZE
+//! Size of heap hashmap
+#define HEAP_ARRAY_SIZE 47
+#endif
+#ifndef ENABLE_THREAD_CACHE
+//! Enable per-thread cache
+#define ENABLE_THREAD_CACHE 1
+#endif
+#ifndef ENABLE_GLOBAL_CACHE
+//! Enable global cache shared between all threads, requires thread cache
+#define ENABLE_GLOBAL_CACHE 1
+#endif
+#ifndef ENABLE_VALIDATE_ARGS
+//! Enable validation of args to public entry points
+#define ENABLE_VALIDATE_ARGS 0
+#endif
+#ifndef ENABLE_STATISTICS
+//! Enable statistics collection
+#define ENABLE_STATISTICS 0
+#endif
+#ifndef ENABLE_ASSERTS
+//! Enable asserts
+#define ENABLE_ASSERTS 0
+#endif
+#ifndef ENABLE_OVERRIDE
+//! Override standard library malloc/free and new/delete entry points
+#define ENABLE_OVERRIDE 0
+#endif
+#ifndef ENABLE_PRELOAD
+//! Support preloading
+#define ENABLE_PRELOAD 0
+#endif
+#ifndef DISABLE_UNMAP
+//! Disable unmapping memory pages (also enables unlimited cache)
+#define DISABLE_UNMAP 0
+#endif
+#ifndef ENABLE_UNLIMITED_CACHE
+//! Enable unlimited global cache (no unmapping until finalization)
+#define ENABLE_UNLIMITED_CACHE 0
+#endif
+#ifndef ENABLE_ADAPTIVE_THREAD_CACHE
+//! Enable adaptive thread cache size based on use heuristics
+#define ENABLE_ADAPTIVE_THREAD_CACHE 0
+#endif
+#ifndef DEFAULT_SPAN_MAP_COUNT
+//! Default number of spans to map in call to map more virtual memory (default
+//! values yield 4MiB here)
+#define DEFAULT_SPAN_MAP_COUNT 64
+#endif
+#ifndef GLOBAL_CACHE_MULTIPLIER
+//! Multiplier for global cache
+#define GLOBAL_CACHE_MULTIPLIER 8
+#endif
+
+#if DISABLE_UNMAP && !ENABLE_GLOBAL_CACHE
+#error Must use global cache if unmap is disabled
+#endif
+
+#if DISABLE_UNMAP
+#undef ENABLE_UNLIMITED_CACHE
+#define ENABLE_UNLIMITED_CACHE 1
+#endif
+
+#if !ENABLE_GLOBAL_CACHE
+#undef ENABLE_UNLIMITED_CACHE
+#define ENABLE_UNLIMITED_CACHE 0
+#endif
+
+#if !ENABLE_THREAD_CACHE
+#undef ENABLE_ADAPTIVE_THREAD_CACHE
+#define ENABLE_ADAPTIVE_THREAD_CACHE 0
+#endif
+
+#if defined(_WIN32) || defined(__WIN32__) || defined(_WIN64)
+#define PLATFORM_WINDOWS 1
+#define PLATFORM_POSIX 0
+#else
+#define PLATFORM_WINDOWS 0
+#define PLATFORM_POSIX 1
+#endif
+
+/// Platform and arch specifics
+#if defined(_MSC_VER) && !defined(__clang__)
+#pragma warning(disable : 5105)
+#ifndef FORCEINLINE
+#define FORCEINLINE inline __forceinline
+#endif
+#define _Static_assert static_assert
+#else
+#ifndef FORCEINLINE
+#define FORCEINLINE inline __attribute__((__always_inline__))
+#endif
+#endif
+#if PLATFORM_WINDOWS
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#include <windows.h>
+#if ENABLE_VALIDATE_ARGS
+#include <intsafe.h>
+#endif
+#else
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include <unistd.h>
+#if defined(__linux__) || defined(__ANDROID__)
+#include <sys/prctl.h>
+#if !defined(PR_SET_VMA)
+#define PR_SET_VMA 0x53564d41
+#define PR_SET_VMA_ANON_NAME 0
+#endif
+#endif
+#if defined(__APPLE__)
+#include <TargetConditionals.h>
+#if !TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
+#include <mach/mach_vm.h>
+#include <mach/vm_statistics.h>
+#endif
+#include <pthread.h>
+#endif
+#if defined(__HAIKU__) || defined(__TINYC__)
+#include <pthread.h>
+#endif
+#endif
+
+#include <errno.h>
+#include <stdint.h>
+#include <string.h>
+
+#if defined(_WIN32) && (!defined(BUILD_DYNAMIC_LINK) || !BUILD_DYNAMIC_LINK)
+#include <fibersapi.h>
+static DWORD fls_key;
+#endif
+
+#if PLATFORM_POSIX
+#include <sched.h>
+#include <sys/mman.h>
+#ifdef __FreeBSD__
+#include <sys/sysctl.h>
+#define MAP_HUGETLB MAP_ALIGNED_SUPER
+#ifndef PROT_MAX
+#define PROT_MAX(f) 0
+#endif
+#else
+#define PROT_MAX(f) 0
+#endif
+#ifdef __sun
+extern int madvise(caddr_t, size_t, int);
+#endif
+#ifndef MAP_UNINITIALIZED
+#define MAP_UNINITIALIZED 0
+#endif
+#endif
+#include <errno.h>
+
+#if ENABLE_ASSERTS
+#undef NDEBUG
+#if defined(_MSC_VER) && !defined(_DEBUG)
+#define _DEBUG
+#endif
+#include <assert.h>
+#define RPMALLOC_TOSTRING_M(x) #x
+#define RPMALLOC_TOSTRING(x) RPMALLOC_TOSTRING_M(x)
+#define rpmalloc_assert(truth, message) \
+ do { \
+ if (!(truth)) { \
+ if (_memory_config.error_callback) { \
+ _memory_config.error_callback(message " (" RPMALLOC_TOSTRING( \
+ truth) ") at " __FILE__ ":" RPMALLOC_TOSTRING(__LINE__)); \
+ } else { \
+ assert((truth) && message); \
+ } \
+ } \
+ } while (0)
+#else
+#define rpmalloc_assert(truth, message) \
+ do { \
+ } while (0)
+#endif
+#if ENABLE_STATISTICS
+#include <stdio.h>
+#endif
+
+//////
+///
+/// Atomic access abstraction (since MSVC does not do C11 yet)
+///
+//////
+
+#if defined(_MSC_VER) && !defined(__clang__)
+
+typedef volatile long atomic32_t;
+typedef volatile long long atomic64_t;
+typedef volatile void *atomicptr_t;
+
+static FORCEINLINE int32_t atomic_load32(atomic32_t *src) { return *src; }
+static FORCEINLINE void atomic_store32(atomic32_t *dst, int32_t val) {
+ *dst = val;
+}
+static FORCEINLINE int32_t atomic_incr32(atomic32_t *val) {
+ return (int32_t)InterlockedIncrement(val);
+}
+static FORCEINLINE int32_t atomic_decr32(atomic32_t *val) {
+ return (int32_t)InterlockedDecrement(val);
+}
+static FORCEINLINE int32_t atomic_add32(atomic32_t *val, int32_t add) {
+ return (int32_t)InterlockedExchangeAdd(val, add) + add;
+}
+static FORCEINLINE int atomic_cas32_acquire(atomic32_t *dst, int32_t val,
+ int32_t ref) {
+ return (InterlockedCompareExchange(dst, val, ref) == ref) ? 1 : 0;
+}
+static FORCEINLINE void atomic_store32_release(atomic32_t *dst, int32_t val) {
+ *dst = val;
+}
+static FORCEINLINE int64_t atomic_load64(atomic64_t *src) { return *src; }
+static FORCEINLINE int64_t atomic_add64(atomic64_t *val, int64_t add) {
+ return (int64_t)InterlockedExchangeAdd64(val, add) + add;
+}
+static FORCEINLINE void *atomic_load_ptr(atomicptr_t *src) {
+ return (void *)*src;
+}
+static FORCEINLINE void atomic_store_ptr(atomicptr_t *dst, void *val) {
+ *dst = val;
+}
+static FORCEINLINE void atomic_store_ptr_release(atomicptr_t *dst, void *val) {
+ *dst = val;
+}
+static FORCEINLINE void *atomic_exchange_ptr_acquire(atomicptr_t *dst,
+ void *val) {
+ return (void *)InterlockedExchangePointer((void *volatile *)dst, val);
+}
+static FORCEINLINE int atomic_cas_ptr(atomicptr_t *dst, void *val, void *ref) {
+ return (InterlockedCompareExchangePointer((void *volatile *)dst, val, ref) ==
+ ref)
+ ? 1
+ : 0;
+}
+
+#define EXPECTED(x) (x)
+#define UNEXPECTED(x) (x)
+
+#else
+
+#include <stdatomic.h>
+
+typedef volatile _Atomic(int32_t) atomic32_t;
+typedef volatile _Atomic(int64_t) atomic64_t;
+typedef volatile _Atomic(void *) atomicptr_t;
+
+static FORCEINLINE int32_t atomic_load32(atomic32_t *src) {
+ return atomic_load_explicit(src, memory_order_relaxed);
+}
+static FORCEINLINE void atomic_store32(atomic32_t *dst, int32_t val) {
+ atomic_store_explicit(dst, val, memory_order_relaxed);
+}
+static FORCEINLINE int32_t atomic_incr32(atomic32_t *val) {
+ return atomic_fetch_add_explicit(val, 1, memory_order_relaxed) + 1;
+}
+static FORCEINLINE int32_t atomic_decr32(atomic32_t *val) {
+ return atomic_fetch_add_explicit(val, -1, memory_order_relaxed) - 1;
+}
+static FORCEINLINE int32_t atomic_add32(atomic32_t *val, int32_t add) {
+ return atomic_fetch_add_explicit(val, add, memory_order_relaxed) + add;
+}
+static FORCEINLINE int atomic_cas32_acquire(atomic32_t *dst, int32_t val,
+ int32_t ref) {
+ return atomic_compare_exchange_weak_explicit(
+ dst, &ref, val, memory_order_acquire, memory_order_relaxed);
+}
+static FORCEINLINE void atomic_store32_release(atomic32_t *dst, int32_t val) {
+ atomic_store_explicit(dst, val, memory_order_release);
+}
+static FORCEINLINE int64_t atomic_load64(atomic64_t *val) {
+ return atomic_load_explicit(val, memory_order_relaxed);
+}
+static FORCEINLINE int64_t atomic_add64(atomic64_t *val, int64_t add) {
+ return atomic_fetch_add_explicit(val, add, memory_order_relaxed) + add;
+}
+static FORCEINLINE void *atomic_load_ptr(atomicptr_t *src) {
+ return atomic_load_explicit(src, memory_order_relaxed);
+}
+static FORCEINLINE void atomic_store_ptr(atomicptr_t *dst, void *val) {
+ atomic_store_explicit(dst, val, memory_order_relaxed);
+}
+static FORCEINLINE void atomic_store_ptr_release(atomicptr_t *dst, void *val) {
+ atomic_store_explicit(dst, val, memory_order_release);
+}
+static FORCEINLINE void *atomic_exchange_ptr_acquire(atomicptr_t *dst,
+ void *val) {
+ return atomic_exchange_explicit(dst, val, memory_order_acquire);
+}
+static FORCEINLINE int atomic_cas_ptr(atomicptr_t *dst, void *val, void *ref) {
+ return atomic_compare_exchange_weak_explicit(
+ dst, &ref, val, memory_order_relaxed, memory_order_relaxed);
+}
+
+#define EXPECTED(x) __builtin_expect((x), 1)
+#define UNEXPECTED(x) __builtin_expect((x), 0)
+
+#endif
+
+////////////
+///
+/// Statistics related functions (evaluate to nothing when statistics not
+/// enabled)
+///
+//////
+
+#if ENABLE_STATISTICS
+#define _rpmalloc_stat_inc(counter) atomic_incr32(counter)
+#define _rpmalloc_stat_dec(counter) atomic_decr32(counter)
+#define _rpmalloc_stat_add(counter, value) \
+ atomic_add32(counter, (int32_t)(value))
+#define _rpmalloc_stat_add64(counter, value) \
+ atomic_add64(counter, (int64_t)(value))
+#define _rpmalloc_stat_add_peak(counter, value, peak) \
+ do { \
+ int32_t _cur_count = atomic_add32(counter, (int32_t)(value)); \
+ if (_cur_count > (peak)) \
+ peak = _cur_count; \
+ } while (0)
+#define _rpmalloc_stat_sub(counter, value) \
+ atomic_add32(counter, -(int32_t)(value))
+#define _rpmalloc_stat_inc_alloc(heap, class_idx) \
+ do { \
+ int32_t alloc_current = \
+ atomic_incr32(&heap->size_class_use[class_idx].alloc_current); \
+ if (alloc_current > heap->size_class_use[class_idx].alloc_peak) \
+ heap->size_class_use[class_idx].alloc_peak = alloc_current; \
+ atomic_incr32(&heap->size_class_use[class_idx].alloc_total); \
+ } while (0)
+#define _rpmalloc_stat_inc_free(heap, class_idx) \
+ do { \
+ atomic_decr32(&heap->size_class_use[class_idx].alloc_current); \
+ atomic_incr32(&heap->size_class_use[class_idx].free_total); \
+ } while (0)
+#else
+#define _rpmalloc_stat_inc(counter) \
+ do { \
+ } while (0)
+#define _rpmalloc_stat_dec(counter) \
+ do { \
+ } while (0)
+#define _rpmalloc_stat_add(counter, value) \
+ do { \
+ } while (0)
+#define _rpmalloc_stat_add64(counter, value) \
+ do { \
+ } while (0)
+#define _rpmalloc_stat_add_peak(counter, value, peak) \
+ do { \
+ } while (0)
+#define _rpmalloc_stat_sub(counter, value) \
+ do { \
+ } while (0)
+#define _rpmalloc_stat_inc_alloc(heap, class_idx) \
+ do { \
+ } while (0)
+#define _rpmalloc_stat_inc_free(heap, class_idx) \
+ do { \
+ } while (0)
+#endif
+
+///
+/// Preconfigured limits and sizes
+///
+
+//! Granularity of a small allocation block (must be power of two)
+#define SMALL_GRANULARITY 16
+//! Small granularity shift count
+#define SMALL_GRANULARITY_SHIFT 4
+//! Number of small block size classes
+#define SMALL_CLASS_COUNT 65
+//! Maximum size of a small block
+#define SMALL_SIZE_LIMIT (SMALL_GRANULARITY * (SMALL_CLASS_COUNT - 1))
+//! Granularity of a medium allocation block
+#define MEDIUM_GRANULARITY 512
+//! Medium granularity shift count
+#define MEDIUM_GRANULARITY_SHIFT 9
+//! Number of medium block size classes
+#define MEDIUM_CLASS_COUNT 61
+//! Total number of small + medium size classes
+#define SIZE_CLASS_COUNT (SMALL_CLASS_COUNT + MEDIUM_CLASS_COUNT)
+//! Number of large block size classes
+#define LARGE_CLASS_COUNT 63
+//! Maximum size of a medium block
+#define MEDIUM_SIZE_LIMIT \
+ (SMALL_SIZE_LIMIT + (MEDIUM_GRANULARITY * MEDIUM_CLASS_COUNT))
+//! Maximum size of a large block
+#define LARGE_SIZE_LIMIT \
+ ((LARGE_CLASS_COUNT * _memory_span_size) - SPAN_HEADER_SIZE)
+//! Size of a span header (must be a multiple of SMALL_GRANULARITY and a power
+//! of two)
+#define SPAN_HEADER_SIZE 128
+//! Number of spans in thread cache
+#define MAX_THREAD_SPAN_CACHE 400
+//! Number of spans to transfer between thread and global cache
+#define THREAD_SPAN_CACHE_TRANSFER 64
+//! Number of spans in thread cache for large spans (must be greater than
+//! LARGE_CLASS_COUNT / 2)
+#define MAX_THREAD_SPAN_LARGE_CACHE 100
+//! Number of spans to transfer between thread and global cache for large spans
+#define THREAD_SPAN_LARGE_CACHE_TRANSFER 6
+
+_Static_assert((SMALL_GRANULARITY & (SMALL_GRANULARITY - 1)) == 0,
+ "Small granularity must be power of two");
+_Static_assert((SPAN_HEADER_SIZE & (SPAN_HEADER_SIZE - 1)) == 0,
+ "Span header size must be power of two");
+
+#if ENABLE_VALIDATE_ARGS
+//! Maximum allocation size to avoid integer overflow
+#undef MAX_ALLOC_SIZE
+#define MAX_ALLOC_SIZE (((size_t)-1) - _memory_span_size)
+#endif
+
+#define pointer_offset(ptr, ofs) (void *)((char *)(ptr) + (ptrdiff_t)(ofs))
+#define pointer_diff(first, second) \
+ (ptrdiff_t)((const char *)(first) - (const char *)(second))
+
+#define INVALID_POINTER ((void *)((uintptr_t)-1))
+
+#define SIZE_CLASS_LARGE SIZE_CLASS_COUNT
+#define SIZE_CLASS_HUGE ((uint32_t)-1)
+
+////////////
+///
+/// Data types
+///
+//////
+
+//! A memory heap, per thread
+typedef struct heap_t heap_t;
+//! Span of memory pages
+typedef struct span_t span_t;
+//! Span list
+typedef struct span_list_t span_list_t;
+//! Span active data
+typedef struct span_active_t span_active_t;
+//! Size class definition
+typedef struct size_class_t size_class_t;
+//! Global cache
+typedef struct global_cache_t global_cache_t;
+
+//! Flag indicating span is the first (master) span of a split superspan
+#define SPAN_FLAG_MASTER 1U
+//! Flag indicating span is a secondary (sub) span of a split superspan
+#define SPAN_FLAG_SUBSPAN 2U
+//! Flag indicating span has blocks with increased alignment
+#define SPAN_FLAG_ALIGNED_BLOCKS 4U
+//! Flag indicating an unmapped master span
+#define SPAN_FLAG_UNMAPPED_MASTER 8U
+
+#if ENABLE_ADAPTIVE_THREAD_CACHE || ENABLE_STATISTICS
+struct span_use_t {
+ //! Current number of spans used (actually used, not in cache)
+ atomic32_t current;
+ //! High water mark of spans used
+ atomic32_t high;
+#if ENABLE_STATISTICS
+ //! Number of spans in deferred list
+ atomic32_t spans_deferred;
+ //! Number of spans transitioned to global cache
+ atomic32_t spans_to_global;
+ //! Number of spans transitioned from global cache
+ atomic32_t spans_from_global;
+ //! Number of spans transitioned to thread cache
+ atomic32_t spans_to_cache;
+ //! Number of spans transitioned from thread cache
+ atomic32_t spans_from_cache;
+ //! Number of spans transitioned to reserved state
+ atomic32_t spans_to_reserved;
+ //! Number of spans transitioned from reserved state
+ atomic32_t spans_from_reserved;
+ //! Number of raw memory map calls
+ atomic32_t spans_map_calls;
+#endif
+};
+typedef struct span_use_t span_use_t;
+#endif
+
+#if ENABLE_STATISTICS
+struct size_class_use_t {
+ //! Current number of allocations
+ atomic32_t alloc_current;
+ //! Peak number of allocations
+ int32_t alloc_peak;
+ //! Total number of allocations
+ atomic32_t alloc_total;
+ //! Total number of frees
+ atomic32_t free_total;
+ //! Number of spans in use
+ atomic32_t spans_current;
+ //! Number of spans transitioned to cache
+ int32_t spans_peak;
+ //! Number of spans transitioned to cache
+ atomic32_t spans_to_cache;
+ //! Number of spans transitioned from cache
+ atomic32_t spans_from_cache;
+ //! Number of spans transitioned from reserved state
+ atomic32_t spans_from_reserved;
+ //! Number of spans mapped
+ atomic32_t spans_map_calls;
+ int32_t unused;
+};
+typedef struct size_class_use_t size_class_use_t;
+#endif
+
+// A span can either represent a single span of memory pages with size declared
+// by span_map_count configuration variable, or a set of spans in a continuous
+// region, a super span. Any reference to the term "span" usually refers to both
+// a single span or a super span. A super span can further be divided into
+// multiple spans (or this, super spans), where the first (super)span is the
+// master and subsequent (super)spans are subspans. The master span keeps track
+// of how many subspans that are still alive and mapped in virtual memory, and
+// once all subspans and master have been unmapped the entire superspan region
+// is released and unmapped (on Windows for example, the entire superspan range
+// has to be released in the same call to release the virtual memory range, but
+// individual subranges can be decommitted individually to reduce physical
+// memory use).
+struct span_t {
+ //! Free list
+ void *free_list;
+ //! Total block count of size class
+ uint32_t block_count;
+ //! Size class
+ uint32_t size_class;
+ //! Index of last block initialized in free list
+ uint32_t free_list_limit;
+ //! Number of used blocks remaining when in partial state
+ uint32_t used_count;
+ //! Deferred free list
+ atomicptr_t free_list_deferred;
+ //! Size of deferred free list, or list of spans when part of a cache list
+ uint32_t list_size;
+ //! Size of a block
+ uint32_t block_size;
+ //! Flags and counters
+ uint32_t flags;
+ //! Number of spans
+ uint32_t span_count;
+ //! Total span counter for master spans
+ uint32_t total_spans;
+ //! Offset from master span for subspans
+ uint32_t offset_from_master;
+ //! Remaining span counter, for master spans
+ atomic32_t remaining_spans;
+ //! Alignment offset
+ uint32_t align_offset;
+ //! Owning heap
+ heap_t *heap;
+ //! Next span
+ span_t *next;
+ //! Previous span
+ span_t *prev;
+};
+_Static_assert(sizeof(span_t) <= SPAN_HEADER_SIZE, "span size mismatch");
+
+struct span_cache_t {
+ size_t count;
+ span_t *span[MAX_THREAD_SPAN_CACHE];
+};
+typedef struct span_cache_t span_cache_t;
+
+struct span_large_cache_t {
+ size_t count;
+ span_t *span[MAX_THREAD_SPAN_LARGE_CACHE];
+};
+typedef struct span_large_cache_t span_large_cache_t;
+
+struct heap_size_class_t {
+ //! Free list of active span
+ void *free_list;
+ //! Double linked list of partially used spans with free blocks.
+ // Previous span pointer in head points to tail span of list.
+ span_t *partial_span;
+ //! Early level cache of fully free spans
+ span_t *cache;
+};
+typedef struct heap_size_class_t heap_size_class_t;
+
+// Control structure for a heap, either a thread heap or a first class heap if
+// enabled
+struct heap_t {
+ //! Owning thread ID
+ uintptr_t owner_thread;
+ //! Free lists for each size class
+ heap_size_class_t size_class[SIZE_CLASS_COUNT];
+#if ENABLE_THREAD_CACHE
+ //! Arrays of fully freed spans, single span
+ span_cache_t span_cache;
+#endif
+ //! List of deferred free spans (single linked list)
+ atomicptr_t span_free_deferred;
+ //! Number of full spans
+ size_t full_span_count;
+ //! Mapped but unused spans
+ span_t *span_reserve;
+ //! Master span for mapped but unused spans
+ span_t *span_reserve_master;
+ //! Number of mapped but unused spans
+ uint32_t spans_reserved;
+ //! Child count
+ atomic32_t child_count;
+ //! Next heap in id list
+ heap_t *next_heap;
+ //! Next heap in orphan list
+ heap_t *next_orphan;
+ //! Heap ID
+ int32_t id;
+ //! Finalization state flag
+ int finalize;
+ //! Master heap owning the memory pages
+ heap_t *master_heap;
+#if ENABLE_THREAD_CACHE
+ //! Arrays of fully freed spans, large spans with > 1 span count
+ span_large_cache_t span_large_cache[LARGE_CLASS_COUNT - 1];
+#endif
+#if RPMALLOC_FIRST_CLASS_HEAPS
+ //! Double linked list of fully utilized spans with free blocks for each size
+ //! class.
+ // Previous span pointer in head points to tail span of list.
+ span_t *full_span[SIZE_CLASS_COUNT];
+ //! Double linked list of large and huge spans allocated by this heap
+ span_t *large_huge_span;
+#endif
+#if ENABLE_ADAPTIVE_THREAD_CACHE || ENABLE_STATISTICS
+ //! Current and high water mark of spans used per span count
+ span_use_t span_use[LARGE_CLASS_COUNT];
+#endif
+#if ENABLE_STATISTICS
+ //! Allocation stats per size class
+ size_class_use_t size_class_use[SIZE_CLASS_COUNT + 1];
+ //! Number of bytes transitioned thread -> global
+ atomic64_t thread_to_global;
+ //! Number of bytes transitioned global -> thread
+ atomic64_t global_to_thread;
+#endif
+};
+
+// Size class for defining a block size bucket
+struct size_class_t {
+ //! Size of blocks in this class
+ uint32_t block_size;
+ //! Number of blocks in each chunk
+ uint16_t block_count;
+ //! Class index this class is merged with
+ uint16_t class_idx;
+};
+_Static_assert(sizeof(size_class_t) == 8, "Size class size mismatch");
+
+struct global_cache_t {
+ //! Cache lock
+ atomic32_t lock;
+ //! Cache count
+ uint32_t count;
+#if ENABLE_STATISTICS
+ //! Insert count
+ size_t insert_count;
+ //! Extract count
+ size_t extract_count;
+#endif
+ //! Cached spans
+ span_t *span[GLOBAL_CACHE_MULTIPLIER * MAX_THREAD_SPAN_CACHE];
+ //! Unlimited cache overflow
+ span_t *overflow;
+};
+
+////////////
+///
+/// Global data
+///
+//////
+
+//! Default span size (64KiB)
+#define _memory_default_span_size (64 * 1024)
+#define _memory_default_span_size_shift 16
+#define _memory_default_span_mask (~((uintptr_t)(_memory_span_size - 1)))
+
+//! Initialized flag
+static int _rpmalloc_initialized;
+//! Main thread ID
+static uintptr_t _rpmalloc_main_thread_id;
+//! Configuration
+static rpmalloc_config_t _memory_config;
+//! Memory page size
+static size_t _memory_page_size;
+//! Shift to divide by page size
+static size_t _memory_page_size_shift;
+//! Granularity at which memory pages are mapped by OS
+static size_t _memory_map_granularity;
+#if RPMALLOC_CONFIGURABLE
+//! Size of a span of memory pages
+static size_t _memory_span_size;
+//! Shift to divide by span size
+static size_t _memory_span_size_shift;
+//! Mask to get to start of a memory span
+static uintptr_t _memory_span_mask;
+#else
+//! Hardwired span size
+#define _memory_span_size _memory_default_span_size
+#define _memory_span_size_shift _memory_default_span_size_shift
+#define _memory_span_mask _memory_default_span_mask
+#endif
+//! Number of spans to map in each map call
+static size_t _memory_span_map_count;
+//! Number of spans to keep reserved in each heap
+static size_t _memory_heap_reserve_count;
+//! Global size classes
+static size_class_t _memory_size_class[SIZE_CLASS_COUNT];
+//! Run-time size limit of medium blocks
+static size_t _memory_medium_size_limit;
+//! Heap ID counter
+static atomic32_t _memory_heap_id;
+//! Huge page support
+static int _memory_huge_pages;
+#if ENABLE_GLOBAL_CACHE
+//! Global span cache
+static global_cache_t _memory_span_cache[LARGE_CLASS_COUNT];
+#endif
+//! Global reserved spans
+static span_t *_memory_global_reserve;
+//! Global reserved count
+static size_t _memory_global_reserve_count;
+//! Global reserved master
+static span_t *_memory_global_reserve_master;
+//! All heaps
+static heap_t *_memory_heaps[HEAP_ARRAY_SIZE];
+//! Used to restrict access to mapping memory for huge pages
+static atomic32_t _memory_global_lock;
+//! Orphaned heaps
+static heap_t *_memory_orphan_heaps;
+#if RPMALLOC_FIRST_CLASS_HEAPS
+//! Orphaned heaps (first class heaps)
+static heap_t *_memory_first_class_orphan_heaps;
+#endif
+#if ENABLE_STATISTICS
+//! Allocations counter
+static atomic64_t _allocation_counter;
+//! Deallocations counter
+static atomic64_t _deallocation_counter;
+//! Active heap count
+static atomic32_t _memory_active_heaps;
+//! Number of currently mapped memory pages
+static atomic32_t _mapped_pages;
+//! Peak number of concurrently mapped memory pages
+static int32_t _mapped_pages_peak;
+//! Number of mapped master spans
+static atomic32_t _master_spans;
+//! Number of unmapped dangling master spans
+static atomic32_t _unmapped_master_spans;
+//! Running counter of total number of mapped memory pages since start
+static atomic32_t _mapped_total;
+//! Running counter of total number of unmapped memory pages since start
+static atomic32_t _unmapped_total;
+//! Number of currently mapped memory pages in OS calls
+static atomic32_t _mapped_pages_os;
+//! Number of currently allocated pages in huge allocations
+static atomic32_t _huge_pages_current;
+//! Peak number of currently allocated pages in huge allocations
+static int32_t _huge_pages_peak;
+#endif
+
+////////////
+///
+/// Thread local heap and ID
+///
+//////
+
+//! Current thread heap
+#if ((defined(__APPLE__) || defined(__HAIKU__)) && ENABLE_PRELOAD) || \
+ defined(__TINYC__)
+static pthread_key_t _memory_thread_heap;
+#else
+#ifdef _MSC_VER
+#define _Thread_local __declspec(thread)
+#define TLS_MODEL
+#else
+#ifndef __HAIKU__
+#define TLS_MODEL __attribute__((tls_model("initial-exec")))
+#else
+#define TLS_MODEL
+#endif
+#if !defined(__clang__) && defined(__GNUC__)
+#define _Thread_local __thread
+#endif
+#endif
+static _Thread_local heap_t *_memory_thread_heap TLS_MODEL;
+#endif
+
+static inline heap_t *get_thread_heap_raw(void) {
+#if (defined(__APPLE__) || defined(__HAIKU__)) && ENABLE_PRELOAD
+ return pthread_getspecific(_memory_thread_heap);
+#else
+ return _memory_thread_heap;
+#endif
+}
+
+//! Get the current thread heap
+static inline heap_t *get_thread_heap(void) {
+ heap_t *heap = get_thread_heap_raw();
+#if ENABLE_PRELOAD
+ if (EXPECTED(heap != 0))
+ return heap;
+ rpmalloc_initialize();
+ return get_thread_heap_raw();
+#else
+ return heap;
+#endif
+}
+
+//! Fast thread ID
+static inline uintptr_t get_thread_id(void) {
+#if defined(_WIN32)
+ return (uintptr_t)((void *)NtCurrentTeb());
+#elif (defined(__GNUC__) || defined(__clang__)) && !defined(__CYGWIN__)
+ uintptr_t tid;
+#if defined(__i386__)
+ __asm__("movl %%gs:0, %0" : "=r"(tid) : :);
+#elif defined(__x86_64__)
+#if defined(__MACH__)
+ __asm__("movq %%gs:0, %0" : "=r"(tid) : :);
+#else
+ __asm__("movq %%fs:0, %0" : "=r"(tid) : :);
+#endif
+#elif defined(__arm__)
+ __asm__ volatile("mrc p15, 0, %0, c13, c0, 3" : "=r"(tid));
+#elif defined(__aarch64__)
+#if defined(__MACH__)
+ // tpidr_el0 likely unused, always return 0 on iOS
+ __asm__ volatile("mrs %0, tpidrro_el0" : "=r"(tid));
+#else
+ __asm__ volatile("mrs %0, tpidr_el0" : "=r"(tid));
+#endif
+#else
+ tid = (uintptr_t)((void *)get_thread_heap_raw());
+#endif
+ return tid;
+#else
+ return (uintptr_t)((void *)get_thread_heap_raw());
+#endif
+}
+
+//! Set the current thread heap
+static void set_thread_heap(heap_t *heap) {
+#if ((defined(__APPLE__) || defined(__HAIKU__)) && ENABLE_PRELOAD) || \
+ defined(__TINYC__)
+ pthread_setspecific(_memory_thread_heap, heap);
+#else
+ _memory_thread_heap = heap;
+#endif
+ if (heap)
+ heap->owner_thread = get_thread_id();
+}
+
+//! Set main thread ID
+extern void rpmalloc_set_main_thread(void);
+
+void rpmalloc_set_main_thread(void) {
+ _rpmalloc_main_thread_id = get_thread_id();
+}
+
+static void _rpmalloc_spin(void) {
+#if defined(_MSC_VER)
+ _mm_pause();
+#elif defined(__x86_64__) || defined(__i386__)
+ __asm__ volatile("pause" ::: "memory");
+#elif defined(__aarch64__) || (defined(__arm__) && __ARM_ARCH >= 7)
+ __asm__ volatile("yield" ::: "memory");
+#elif defined(__powerpc__) || defined(__powerpc64__)
+ // No idea if ever been compiled in such archs but ... as precaution
+ __asm__ volatile("or 27,27,27");
+#elif defined(__sparc__)
+ __asm__ volatile("rd %ccr, %g0 \n\trd %ccr, %g0 \n\trd %ccr, %g0");
+#else
+ struct timespec ts = {0};
+ nanosleep(&ts, 0);
+#endif
+}
+
+#if defined(_WIN32) && (!defined(BUILD_DYNAMIC_LINK) || !BUILD_DYNAMIC_LINK)
+static void NTAPI _rpmalloc_thread_destructor(void *value) {
+#if ENABLE_OVERRIDE
+ // If this is called on main thread it means rpmalloc_finalize
+ // has not been called and shutdown is forced (through _exit) or unclean
+ if (get_thread_id() == _rpmalloc_main_thread_id)
+ return;
+#endif
+ if (value)
+ rpmalloc_thread_finalize(1);
+}
+#endif
+
+////////////
+///
+/// Low level memory map/unmap
+///
+//////
+
+static void _rpmalloc_set_name(void *address, size_t size) {
+#if defined(__linux__) || defined(__ANDROID__)
+ const char *name = _memory_huge_pages ? _memory_config.huge_page_name
+ : _memory_config.page_name;
+ if (address == MAP_FAILED || !name)
+ return;
+ // If the kernel does not support CONFIG_ANON_VMA_NAME or if the call fails
+ // (e.g. invalid name) it is a no-op basically.
+ (void)prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, (uintptr_t)address, size,
+ (uintptr_t)name);
+#else
+ (void)sizeof(size);
+ (void)sizeof(address);
+#endif
+}
+
+//! Map more virtual memory
+// size is number of bytes to map
+// offset receives the offset in bytes from start of mapped region
+// returns address to start of mapped region to use
+static void *_rpmalloc_mmap(size_t size, size_t *offset) {
+ rpmalloc_assert(!(size % _memory_page_size), "Invalid mmap size");
+ rpmalloc_assert(size >= _memory_page_size, "Invalid mmap size");
+ void *address = _memory_config.memory_map(size, offset);
+ if (EXPECTED(address != 0)) {
+ _rpmalloc_stat_add_peak(&_mapped_pages, (size >> _memory_page_size_shift),
+ _mapped_pages_peak);
+ _rpmalloc_stat_add(&_mapped_total, (size >> _memory_page_size_shift));
+ }
+ return address;
+}
+
+//! Unmap virtual memory
+// address is the memory address to unmap, as returned from _memory_map
+// size is the number of bytes to unmap, which might be less than full region
+// for a partial unmap offset is the offset in bytes to the actual mapped
+// region, as set by _memory_map release is set to 0 for partial unmap, or size
+// of entire range for a full unmap
+static void _rpmalloc_unmap(void *address, size_t size, size_t offset,
+ size_t release) {
+ rpmalloc_assert(!release || (release >= size), "Invalid unmap size");
+ rpmalloc_assert(!release || (release >= _memory_page_size),
+ "Invalid unmap size");
+ if (release) {
+ rpmalloc_assert(!(release % _memory_page_size), "Invalid unmap size");
+ _rpmalloc_stat_sub(&_mapped_pages, (release >> _memory_page_size_shift));
+ _rpmalloc_stat_add(&_unmapped_total, (release >> _memory_page_size_shift));
+ }
+ _memory_config.memory_unmap(address, size, offset, release);
+}
+
+//! Default implementation to map new pages to virtual memory
+static void *_rpmalloc_mmap_os(size_t size, size_t *offset) {
+ // Either size is a heap (a single page) or a (multiple) span - we only need
+ // to align spans, and only if larger than map granularity
+ size_t padding = ((size >= _memory_span_size) &&
+ (_memory_span_size > _memory_map_granularity))
+ ? _memory_span_size
+ : 0;
+ rpmalloc_assert(size >= _memory_page_size, "Invalid mmap size");
+#if PLATFORM_WINDOWS
+ // Ok to MEM_COMMIT - according to MSDN, "actual physical pages are not
+ // allocated unless/until the virtual addresses are actually accessed"
+ void *ptr = VirtualAlloc(0, size + padding,
+ (_memory_huge_pages ? MEM_LARGE_PAGES : 0) |
+ MEM_RESERVE | MEM_COMMIT,
+ PAGE_READWRITE);
+ if (!ptr) {
+ if (_memory_config.map_fail_callback) {
+ if (_memory_config.map_fail_callback(size + padding))
+ return _rpmalloc_mmap_os(size, offset);
+ } else {
+ rpmalloc_assert(ptr, "Failed to map virtual memory block");
+ }
+ return 0;
+ }
+#else
+ int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_UNINITIALIZED;
+#if defined(__APPLE__) && !TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
+ int fd = (int)VM_MAKE_TAG(240U);
+ if (_memory_huge_pages)
+ fd |= VM_FLAGS_SUPERPAGE_SIZE_2MB;
+ void *ptr = mmap(0, size + padding, PROT_READ | PROT_WRITE, flags, fd, 0);
+#elif defined(MAP_HUGETLB)
+ void *ptr = mmap(0, size + padding,
+ PROT_READ | PROT_WRITE | PROT_MAX(PROT_READ | PROT_WRITE),
+ (_memory_huge_pages ? MAP_HUGETLB : 0) | flags, -1, 0);
+#if defined(MADV_HUGEPAGE)
+ // In some configurations, huge pages allocations might fail thus
+ // we fallback to normal allocations and promote the region as transparent
+ // huge page
+ if ((ptr == MAP_FAILED || !ptr) && _memory_huge_pages) {
+ ptr = mmap(0, size + padding, PROT_READ | PROT_WRITE, flags, -1, 0);
+ if (ptr && ptr != MAP_FAILED) {
+ int prm = madvise(ptr, size + padding, MADV_HUGEPAGE);
+ (void)prm;
+ rpmalloc_assert((prm == 0), "Failed to promote the page to THP");
+ }
+ }
+#endif
+ _rpmalloc_set_name(ptr, size + padding);
+#elif defined(MAP_ALIGNED)
+ const size_t align =
+ (sizeof(size_t) * 8) - (size_t)(__builtin_clzl(size - 1));
+ void *ptr =
+ mmap(0, size + padding, PROT_READ | PROT_WRITE,
+ (_memory_huge_pages ? MAP_ALIGNED(align) : 0) | flags, -1, 0);
+#elif defined(MAP_ALIGN)
+ caddr_t base = (_memory_huge_pages ? (caddr_t)(4 << 20) : 0);
+ void *ptr = mmap(base, size + padding, PROT_READ | PROT_WRITE,
+ (_memory_huge_pages ? MAP_ALIGN : 0) | flags, -1, 0);
+#else
+ void *ptr = mmap(0, size + padding, PROT_READ | PROT_WRITE, flags, -1, 0);
+#endif
+ if ((ptr == MAP_FAILED) || !ptr) {
+ if (_memory_config.map_fail_callback) {
+ if (_memory_config.map_fail_callback(size + padding))
+ return _rpmalloc_mmap_os(size, offset);
+ } else if (errno != ENOMEM) {
+ rpmalloc_assert((ptr != MAP_FAILED) && ptr,
+ "Failed to map virtual memory block");
+ }
+ return 0;
+ }
+#endif
+ _rpmalloc_stat_add(&_mapped_pages_os,
+ (int32_t)((size + padding) >> _memory_page_size_shift));
+ if (padding) {
+ size_t final_padding = padding - ((uintptr_t)ptr & ~_memory_span_mask);
+ rpmalloc_assert(final_padding <= _memory_span_size,
+ "Internal failure in padding");
+ rpmalloc_assert(final_padding <= padding, "Internal failure in padding");
+ rpmalloc_assert(!(final_padding % 8), "Internal failure in padding");
+ ptr = pointer_offset(ptr, final_padding);
+ *offset = final_padding >> 3;
+ }
+ rpmalloc_assert((size < _memory_span_size) ||
+ !((uintptr_t)ptr & ~_memory_span_mask),
+ "Internal failure in padding");
+ return ptr;
+}
+
+//! Default implementation to unmap pages from virtual memory
+static void _rpmalloc_unmap_os(void *address, size_t size, size_t offset,
+ size_t release) {
+ rpmalloc_assert(release || (offset == 0), "Invalid unmap size");
+ rpmalloc_assert(!release || (release >= _memory_page_size),
+ "Invalid unmap size");
+ rpmalloc_assert(size >= _memory_page_size, "Invalid unmap size");
+ if (release && offset) {
+ offset <<= 3;
+ address = pointer_offset(address, -(int32_t)offset);
+ if ((release >= _memory_span_size) &&
+ (_memory_span_size > _memory_map_granularity)) {
+ // Padding is always one span size
+ release += _memory_span_size;
+ }
+ }
+#if !DISABLE_UNMAP
+#if PLATFORM_WINDOWS
+ if (!VirtualFree(address, release ? 0 : size,
+ release ? MEM_RELEASE : MEM_DECOMMIT)) {
+ rpmalloc_assert(0, "Failed to unmap virtual memory block");
+ }
+#else
+ if (release) {
+ if (munmap(address, release)) {
+ rpmalloc_assert(0, "Failed to unmap virtual memory block");
+ }
+ } else {
+#if defined(MADV_FREE_REUSABLE)
+ int ret;
+ while ((ret = madvise(address, size, MADV_FREE_REUSABLE)) == -1 &&
+ (errno == EAGAIN))
+ errno = 0;
+ if ((ret == -1) && (errno != 0)) {
+#elif defined(MADV_DONTNEED)
+ if (madvise(address, size, MADV_DONTNEED)) {
+#elif defined(MADV_PAGEOUT)
+ if (madvise(address, size, MADV_PAGEOUT)) {
+#elif defined(MADV_FREE)
+ if (madvise(address, size, MADV_FREE)) {
+#else
+ if (posix_madvise(address, size, POSIX_MADV_DONTNEED)) {
+#endif
+ rpmalloc_assert(0, "Failed to madvise virtual memory block as free");
+ }
+ }
+#endif
+#endif
+ if (release)
+ _rpmalloc_stat_sub(&_mapped_pages_os, release >> _memory_page_size_shift);
+}
+
+static void _rpmalloc_span_mark_as_subspan_unless_master(span_t *master,
+ span_t *subspan,
+ size_t span_count);
+
+//! Use global reserved spans to fulfill a memory map request (reserve size must
+//! be checked by caller)
+static span_t *_rpmalloc_global_get_reserved_spans(size_t span_count) {
+ span_t *span = _memory_global_reserve;
+ _rpmalloc_span_mark_as_subspan_unless_master(_memory_global_reserve_master,
+ span, span_count);
+ _memory_global_reserve_count -= span_count;
+ if (_memory_global_reserve_count)
+ _memory_global_reserve =
+ (span_t *)pointer_offset(span, span_count << _memory_span_size_shift);
+ else
+ _memory_global_reserve = 0;
+ return span;
+}
+
+//! Store the given spans as global reserve (must only be called from within new
+//! heap allocation, not thread safe)
+static void _rpmalloc_global_set_reserved_spans(span_t *master, span_t *reserve,
+ size_t reserve_span_count) {
+ _memory_global_reserve_master = master;
+ _memory_global_reserve_count = reserve_span_count;
+ _memory_global_reserve = reserve;
+}
+
+////////////
+///
+/// Span linked list management
+///
+//////
+
+//! Add a span to double linked list at the head
+static void _rpmalloc_span_double_link_list_add(span_t **head, span_t *span) {
+ if (*head)
+ (*head)->prev = span;
+ span->next = *head;
+ *head = span;
+}
+
+//! Pop head span from double linked list
+static void _rpmalloc_span_double_link_list_pop_head(span_t **head,
+ span_t *span) {
+ rpmalloc_assert(*head == span, "Linked list corrupted");
+ span = *head;
+ *head = span->next;
+}
+
+//! Remove a span from double linked list
+static void _rpmalloc_span_double_link_list_remove(span_t **head,
+ span_t *span) {
+ rpmalloc_assert(*head, "Linked list corrupted");
+ if (*head == span) {
+ *head = span->next;
+ } else {
+ span_t *next_span = span->next;
+ span_t *prev_span = span->prev;
+ prev_span->next = next_span;
+ if (EXPECTED(next_span != 0))
+ next_span->prev = prev_span;
+ }
+}
+
+////////////
+///
+/// Span control
+///
+//////
+
+static void _rpmalloc_heap_cache_insert(heap_t *heap, span_t *span);
+
+static void _rpmalloc_heap_finalize(heap_t *heap);
+
+static void _rpmalloc_heap_set_reserved_spans(heap_t *heap, span_t *master,
+ span_t *reserve,
+ size_t reserve_span_count);
+
+//! Declare the span to be a subspan and store distance from master span and
+//! span count
+static void _rpmalloc_span_mark_as_subspan_unless_master(span_t *master,
+ span_t *subspan,
+ size_t span_count) {
+ rpmalloc_assert((subspan != master) || (subspan->flags & SPAN_FLAG_MASTER),
+ "Span master pointer and/or flag mismatch");
+ if (subspan != master) {
+ subspan->flags = SPAN_FLAG_SUBSPAN;
+ subspan->offset_from_master =
+ (uint32_t)((uintptr_t)pointer_diff(subspan, master) >>
+ _memory_span_size_shift);
+ subspan->align_offset = 0;
+ }
+ subspan->span_count = (uint32_t)span_count;
+}
+
+//! Use reserved spans to fulfill a memory map request (reserve size must be
+//! checked by caller)
+static span_t *_rpmalloc_span_map_from_reserve(heap_t *heap,
+ size_t span_count) {
+ // Update the heap span reserve
+ span_t *span = heap->span_reserve;
+ heap->span_reserve =
+ (span_t *)pointer_offset(span, span_count * _memory_span_size);
+ heap->spans_reserved -= (uint32_t)span_count;
+
+ _rpmalloc_span_mark_as_subspan_unless_master(heap->span_reserve_master, span,
+ span_count);
+ if (span_count <= LARGE_CLASS_COUNT)
+ _rpmalloc_stat_inc(&heap->span_use[span_count - 1].spans_from_reserved);
+
+ return span;
+}
+
+//! Get the aligned number of spans to map in based on wanted count, configured
+//! mapping granularity and the page size
+static size_t _rpmalloc_span_align_count(size_t span_count) {
+ size_t request_count = (span_count > _memory_span_map_count)
+ ? span_count
+ : _memory_span_map_count;
+ if ((_memory_page_size > _memory_span_size) &&
+ ((request_count * _memory_span_size) % _memory_page_size))
+ request_count +=
+ _memory_span_map_count - (request_count % _memory_span_map_count);
+ return request_count;
+}
+
+//! Setup a newly mapped span
+static void _rpmalloc_span_initialize(span_t *span, size_t total_span_count,
+ size_t span_count, size_t align_offset) {
+ span->total_spans = (uint32_t)total_span_count;
+ span->span_count = (uint32_t)span_count;
+ span->align_offset = (uint32_t)align_offset;
+ span->flags = SPAN_FLAG_MASTER;
+ atomic_store32(&span->remaining_spans, (int32_t)total_span_count);
+}
+
+static void _rpmalloc_span_unmap(span_t *span);
+
+//! Map an aligned set of spans, taking configured mapping granularity and the
+//! page size into account
+static span_t *_rpmalloc_span_map_aligned_count(heap_t *heap,
+ size_t span_count) {
+ // If we already have some, but not enough, reserved spans, release those to
+ // heap cache and map a new full set of spans. Otherwise we would waste memory
+ // if page size > span size (huge pages)
+ size_t aligned_span_count = _rpmalloc_span_align_count(span_count);
+ size_t align_offset = 0;
+ span_t *span = (span_t *)_rpmalloc_mmap(
+ aligned_span_count * _memory_span_size, &align_offset);
+ if (!span)
+ return 0;
+ _rpmalloc_span_initialize(span, aligned_span_count, span_count, align_offset);
+ _rpmalloc_stat_inc(&_master_spans);
+ if (span_count <= LARGE_CLASS_COUNT)
+ _rpmalloc_stat_inc(&heap->span_use[span_count - 1].spans_map_calls);
+ if (aligned_span_count > span_count) {
+ span_t *reserved_spans =
+ (span_t *)pointer_offset(span, span_count * _memory_span_size);
+ size_t reserved_count = aligned_span_count - span_count;
+ if (heap->spans_reserved) {
+ _rpmalloc_span_mark_as_subspan_unless_master(
+ heap->span_reserve_master, heap->span_reserve, heap->spans_reserved);
+ _rpmalloc_heap_cache_insert(heap, heap->span_reserve);
+ }
+ if (reserved_count > _memory_heap_reserve_count) {
+ // If huge pages or eager spam map count, the global reserve spin lock is
+ // held by caller, _rpmalloc_span_map
+ rpmalloc_assert(atomic_load32(&_memory_global_lock) == 1,
+ "Global spin lock not held as expected");
+ size_t remain_count = reserved_count - _memory_heap_reserve_count;
+ reserved_count = _memory_heap_reserve_count;
+ span_t *remain_span = (span_t *)pointer_offset(
+ reserved_spans, reserved_count * _memory_span_size);
+ if (_memory_global_reserve) {
+ _rpmalloc_span_mark_as_subspan_unless_master(
+ _memory_global_reserve_master, _memory_global_reserve,
+ _memory_global_reserve_count);
+ _rpmalloc_span_unmap(_memory_global_reserve);
+ }
+ _rpmalloc_global_set_reserved_spans(span, remain_span, remain_count);
+ }
+ _rpmalloc_heap_set_reserved_spans(heap, span, reserved_spans,
+ reserved_count);
+ }
+ return span;
+}
+
+//! Map in memory pages for the given number of spans (or use previously
+//! reserved pages)
+static span_t *_rpmalloc_span_map(heap_t *heap, size_t span_count) {
+ if (span_count <= heap->spans_reserved)
+ return _rpmalloc_span_map_from_reserve(heap, span_count);
+ span_t *span = 0;
+ int use_global_reserve =
+ (_memory_page_size > _memory_span_size) ||
+ (_memory_span_map_count > _memory_heap_reserve_count);
+ if (use_global_reserve) {
+ // If huge pages, make sure only one thread maps more memory to avoid bloat
+ while (!atomic_cas32_acquire(&_memory_global_lock, 1, 0))
+ _rpmalloc_spin();
+ if (_memory_global_reserve_count >= span_count) {
+ size_t reserve_count =
+ (!heap->spans_reserved ? _memory_heap_reserve_count : span_count);
+ if (_memory_global_reserve_count < reserve_count)
+ reserve_count = _memory_global_reserve_count;
+ span = _rpmalloc_global_get_reserved_spans(reserve_count);
+ if (span) {
+ if (reserve_count > span_count) {
+ span_t *reserved_span = (span_t *)pointer_offset(
+ span, span_count << _memory_span_size_shift);
+ _rpmalloc_heap_set_reserved_spans(heap, _memory_global_reserve_master,
+ reserved_span,
+ reserve_count - span_count);
+ }
+ // Already marked as subspan in _rpmalloc_global_get_reserved_spans
+ span->span_count = (uint32_t)span_count;
+ }
+ }
+ }
+ if (!span)
+ span = _rpmalloc_span_map_aligned_count(heap, span_count);
+ if (use_global_reserve)
+ atomic_store32_release(&_memory_global_lock, 0);
+ return span;
+}
+
+//! Unmap memory pages for the given number of spans (or mark as unused if no
+//! partial unmappings)
+static void _rpmalloc_span_unmap(span_t *span) {
+ rpmalloc_assert((span->flags & SPAN_FLAG_MASTER) ||
+ (span->flags & SPAN_FLAG_SUBSPAN),
+ "Span flag corrupted");
+ rpmalloc_assert(!(span->flags & SPAN_FLAG_MASTER) ||
+ !(span->flags & SPAN_FLAG_SUBSPAN),
+ "Span flag corrupted");
+
+ int is_master = !!(span->flags & SPAN_FLAG_MASTER);
+ span_t *master =
+ is_master ? span
+ : ((span_t *)pointer_offset(
+ span, -(intptr_t)((uintptr_t)span->offset_from_master *
+ _memory_span_size)));
+ rpmalloc_assert(is_master || (span->flags & SPAN_FLAG_SUBSPAN),
+ "Span flag corrupted");
+ rpmalloc_assert(master->flags & SPAN_FLAG_MASTER, "Span flag corrupted");
+
+ size_t span_count = span->span_count;
+ if (!is_master) {
+ // Directly unmap subspans (unless huge pages, in which case we defer and
+ // unmap entire page range with master)
+ rpmalloc_assert(span->align_offset == 0, "Span align offset corrupted");
+ if (_memory_span_size >= _memory_page_size)
+ _rpmalloc_unmap(span, span_count * _memory_span_size, 0, 0);
+ } else {
+ // Special double flag to denote an unmapped master
+ // It must be kept in memory since span header must be used
+ span->flags |=
+ SPAN_FLAG_MASTER | SPAN_FLAG_SUBSPAN | SPAN_FLAG_UNMAPPED_MASTER;
+ _rpmalloc_stat_add(&_unmapped_master_spans, 1);
+ }
+
+ if (atomic_add32(&master->remaining_spans, -(int32_t)span_count) <= 0) {
+ // Everything unmapped, unmap the master span with release flag to unmap the
+ // entire range of the super span
+ rpmalloc_assert(!!(master->flags & SPAN_FLAG_MASTER) &&
+ !!(master->flags & SPAN_FLAG_SUBSPAN),
+ "Span flag corrupted");
+ size_t unmap_count = master->span_count;
+ if (_memory_span_size < _memory_page_size)
+ unmap_count = master->total_spans;
+ _rpmalloc_stat_sub(&_master_spans, 1);
+ _rpmalloc_stat_sub(&_unmapped_master_spans, 1);
+ _rpmalloc_unmap(master, unmap_count * _memory_span_size,
+ master->align_offset,
+ (size_t)master->total_spans * _memory_span_size);
+ }
+}
+
+//! Move the span (used for small or medium allocations) to the heap thread
+//! cache
+static void _rpmalloc_span_release_to_cache(heap_t *heap, span_t *span) {
+ rpmalloc_assert(heap == span->heap, "Span heap pointer corrupted");
+ rpmalloc_assert(span->size_class < SIZE_CLASS_COUNT,
+ "Invalid span size class");
+ rpmalloc_assert(span->span_count == 1, "Invalid span count");
+#if ENABLE_ADAPTIVE_THREAD_CACHE || ENABLE_STATISTICS
+ atomic_decr32(&heap->span_use[0].current);
+#endif
+ _rpmalloc_stat_dec(&heap->size_class_use[span->size_class].spans_current);
+ if (!heap->finalize) {
+ _rpmalloc_stat_inc(&heap->span_use[0].spans_to_cache);
+ _rpmalloc_stat_inc(&heap->size_class_use[span->size_class].spans_to_cache);
+ if (heap->size_class[span->size_class].cache)
+ _rpmalloc_heap_cache_insert(heap,
+ heap->size_class[span->size_class].cache);
+ heap->size_class[span->size_class].cache = span;
+ } else {
+ _rpmalloc_span_unmap(span);
+ }
+}
+
+//! Initialize a (partial) free list up to next system memory page, while
+//! reserving the first block as allocated, returning number of blocks in list
+static uint32_t free_list_partial_init(void **list, void **first_block,
+ void *page_start, void *block_start,
+ uint32_t block_count,
+ uint32_t block_size) {
+ rpmalloc_assert(block_count, "Internal failure");
+ *first_block = block_start;
+ if (block_count > 1) {
+ void *free_block = pointer_offset(block_start, block_size);
+ void *block_end =
+ pointer_offset(block_start, (size_t)block_size * block_count);
+ // If block size is less than half a memory page, bound init to next memory
+ // page boundary
+ if (block_size < (_memory_page_size >> 1)) {
+ void *page_end = pointer_offset(page_start, _memory_page_size);
+ if (page_end < block_end)
+ block_end = page_end;
+ }
+ *list = free_block;
+ block_count = 2;
+ void *next_block = pointer_offset(free_block, block_size);
+ while (next_block < block_end) {
+ *((void **)free_block) = next_block;
+ free_block = next_block;
+ ++block_count;
+ next_block = pointer_offset(next_block, block_size);
+ }
+ *((void **)free_block) = 0;
+ } else {
+ *list = 0;
+ }
+ return block_count;
+}
+
+//! Initialize an unused span (from cache or mapped) to be new active span,
+//! putting the initial free list in heap class free list
+static void *_rpmalloc_span_initialize_new(heap_t *heap,
+ heap_size_class_t *heap_size_class,
+ span_t *span, uint32_t class_idx) {
+ rpmalloc_assert(span->span_count == 1, "Internal failure");
+ size_class_t *size_class = _memory_size_class + class_idx;
+ span->size_class = class_idx;
+ span->heap = heap;
+ span->flags &= ~SPAN_FLAG_ALIGNED_BLOCKS;
+ span->block_size = size_class->block_size;
+ span->block_count = size_class->block_count;
+ span->free_list = 0;
+ span->list_size = 0;
+ atomic_store_ptr_release(&span->free_list_deferred, 0);
+
+ // Setup free list. Only initialize one system page worth of free blocks in
+ // list
+ void *block;
+ span->free_list_limit =
+ free_list_partial_init(&heap_size_class->free_list, &block, span,
+ pointer_offset(span, SPAN_HEADER_SIZE),
+ size_class->block_count, size_class->block_size);
+ // Link span as partial if there remains blocks to be initialized as free
+ // list, or full if fully initialized
+ if (span->free_list_limit < span->block_count) {
+ _rpmalloc_span_double_link_list_add(&heap_size_class->partial_span, span);
+ span->used_count = span->free_list_limit;
+ } else {
+#if RPMALLOC_FIRST_CLASS_HEAPS
+ _rpmalloc_span_double_link_list_add(&heap->full_span[class_idx], span);
+#endif
+ ++heap->full_span_count;
+ span->used_count = span->block_count;
+ }
+ return block;
+}
+
+static void _rpmalloc_span_extract_free_list_deferred(span_t *span) {
+ // We need acquire semantics on the CAS operation since we are interested in
+ // the list size Refer to _rpmalloc_deallocate_defer_small_or_medium for
+ // further comments on this dependency
+ do {
+ span->free_list =
+ atomic_exchange_ptr_acquire(&span->free_list_deferred, INVALID_POINTER);
+ } while (span->free_list == INVALID_POINTER);
+ span->used_count -= span->list_size;
+ span->list_size = 0;
+ atomic_store_ptr_release(&span->free_list_deferred, 0);
+}
+
+static int _rpmalloc_span_is_fully_utilized(span_t *span) {
+ rpmalloc_assert(span->free_list_limit <= span->block_count,
+ "Span free list corrupted");
+ return !span->free_list && (span->free_list_limit >= span->block_count);
+}
+
+static int _rpmalloc_span_finalize(heap_t *heap, size_t iclass, span_t *span,
+ span_t **list_head) {
+ void *free_list = heap->size_class[iclass].free_list;
+ span_t *class_span = (span_t *)((uintptr_t)free_list & _memory_span_mask);
+ if (span == class_span) {
+ // Adopt the heap class free list back into the span free list
+ void *block = span->free_list;
+ void *last_block = 0;
+ while (block) {
+ last_block = block;
+ block = *((void **)block);
+ }
+ uint32_t free_count = 0;
+ block = free_list;
+ while (block) {
+ ++free_count;
+ block = *((void **)block);
+ }
+ if (last_block) {
+ *((void **)last_block) = free_list;
+ } else {
+ span->free_list = free_list;
+ }
+ heap->size_class[iclass].free_list = 0;
+ span->used_count -= free_count;
+ }
+ // If this assert triggers you have memory leaks
+ rpmalloc_assert(span->list_size == span->used_count, "Memory leak detected");
+ if (span->list_size == span->used_count) {
+ _rpmalloc_stat_dec(&heap->span_use[0].current);
+ _rpmalloc_stat_dec(&heap->size_class_use[iclass].spans_current);
+ // This function only used for spans in double linked lists
+ if (list_head)
+ _rpmalloc_span_double_link_list_remove(list_head, span);
+ _rpmalloc_span_unmap(span);
+ return 1;
+ }
+ return 0;
+}
+
+////////////
+///
+/// Global cache
+///
+//////
+
+#if ENABLE_GLOBAL_CACHE
+
+//! Finalize a global cache
+static void _rpmalloc_global_cache_finalize(global_cache_t *cache) {
+ while (!atomic_cas32_acquire(&cache->lock, 1, 0))
+ _rpmalloc_spin();
+
+ for (size_t ispan = 0; ispan < cache->count; ++ispan)
+ _rpmalloc_span_unmap(cache->span[ispan]);
+ cache->count = 0;
+
+ while (cache->overflow) {
+ span_t *span = cache->overflow;
+ cache->overflow = span->next;
+ _rpmalloc_span_unmap(span);
+ }
+
+ atomic_store32_release(&cache->lock, 0);
+}
+
+static void _rpmalloc_global_cache_insert_spans(span_t **span,
+ size_t span_count,
+ size_t count) {
+ const size_t cache_limit =
+ (span_count == 1) ? GLOBAL_CACHE_MULTIPLIER * MAX_THREAD_SPAN_CACHE
+ : GLOBAL_CACHE_MULTIPLIER *
+ (MAX_THREAD_SPAN_LARGE_CACHE - (span_count >> 1));
+
+ global_cache_t *cache = &_memory_span_cache[span_count - 1];
+
+ size_t insert_count = count;
+ while (!atomic_cas32_acquire(&cache->lock, 1, 0))
+ _rpmalloc_spin();
+
+#if ENABLE_STATISTICS
+ cache->insert_count += count;
+#endif
+ if ((cache->count + insert_count) > cache_limit)
+ insert_count = cache_limit - cache->count;
+
+ memcpy(cache->span + cache->count, span, sizeof(span_t *) * insert_count);
+ cache->count += (uint32_t)insert_count;
+
+#if ENABLE_UNLIMITED_CACHE
+ while (insert_count < count) {
+#else
+ // Enable unlimited cache if huge pages, or we will leak since it is unlikely
+ // that an entire huge page will be unmapped, and we're unable to partially
+ // decommit a huge page
+ while ((_memory_page_size > _memory_span_size) && (insert_count < count)) {
+#endif
+ span_t *current_span = span[insert_count++];
+ current_span->next = cache->overflow;
+ cache->overflow = current_span;
+ }
+ atomic_store32_release(&cache->lock, 0);
+
+ span_t *keep = 0;
+ for (size_t ispan = insert_count; ispan < count; ++ispan) {
+ span_t *current_span = span[ispan];
+ // Keep master spans that has remaining subspans to avoid dangling them
+ if ((current_span->flags & SPAN_FLAG_MASTER) &&
+ (atomic_load32(¤t_span->remaining_spans) >
+ (int32_t)current_span->span_count)) {
+ current_span->next = keep;
+ keep = current_span;
+ } else {
+ _rpmalloc_span_unmap(current_span);
+ }
+ }
+
+ if (keep) {
+ while (!atomic_cas32_acquire(&cache->lock, 1, 0))
+ _rpmalloc_spin();
+
+ size_t islot = 0;
+ while (keep) {
+ for (; islot < cache->count; ++islot) {
+ span_t *current_span = cache->span[islot];
+ if (!(current_span->flags & SPAN_FLAG_MASTER) ||
+ ((current_span->flags & SPAN_FLAG_MASTER) &&
+ (atomic_load32(¤t_span->remaining_spans) <=
+ (int32_t)current_span->span_count))) {
+ _rpmalloc_span_unmap(current_span);
+ cache->span[islot] = keep;
+ break;
+ }
+ }
+ if (islot == cache->count)
+ break;
+ keep = keep->next;
+ }
+
+ if (keep) {
+ span_t *tail = keep;
+ while (tail->next)
+ tail = tail->next;
+ tail->next = cache->overflow;
+ cache->overflow = keep;
+ }
+
+ atomic_store32_release(&cache->lock, 0);
+ }
+}
+
+static size_t _rpmalloc_global_cache_extract_spans(span_t **span,
+ size_t span_count,
+ size_t count) {
+ global_cache_t *cache = &_memory_span_cache[span_count - 1];
+
+ size_t extract_count = 0;
+ while (!atomic_cas32_acquire(&cache->lock, 1, 0))
+ _rpmalloc_spin();
+
+#if ENABLE_STATISTICS
+ cache->extract_count += count;
+#endif
+ size_t want = count - extract_count;
+ if (want > cache->count)
+ want = cache->count;
+
+ memcpy(span + extract_count, cache->span + (cache->count - want),
+ sizeof(span_t *) * want);
+ cache->count -= (uint32_t)want;
+ extract_count += want;
+
+ while ((extract_count < count) && cache->overflow) {
+ span_t *current_span = cache->overflow;
+ span[extract_count++] = current_span;
+ cache->overflow = current_span->next;
+ }
+
+#if ENABLE_ASSERTS
+ for (size_t ispan = 0; ispan < extract_count; ++ispan) {
+ assert(span[ispan]->span_count == span_count);
+ }
+#endif
+
+ atomic_store32_release(&cache->lock, 0);
+
+ return extract_count;
+}
+
+#endif
+
+////////////
+///
+/// Heap control
+///
+//////
+
+static void _rpmalloc_deallocate_huge(span_t *);
+
+//! Store the given spans as reserve in the given heap
+static void _rpmalloc_heap_set_reserved_spans(heap_t *heap, span_t *master,
+ span_t *reserve,
+ size_t reserve_span_count) {
+ heap->span_reserve_master = master;
+ heap->span_reserve = reserve;
+ heap->spans_reserved = (uint32_t)reserve_span_count;
+}
+
+//! Adopt the deferred span cache list, optionally extracting the first single
+//! span for immediate re-use
+static void _rpmalloc_heap_cache_adopt_deferred(heap_t *heap,
+ span_t **single_span) {
+ span_t *span = (span_t *)((void *)atomic_exchange_ptr_acquire(
+ &heap->span_free_deferred, 0));
+ while (span) {
+ span_t *next_span = (span_t *)span->free_list;
+ rpmalloc_assert(span->heap == heap, "Span heap pointer corrupted");
+ if (EXPECTED(span->size_class < SIZE_CLASS_COUNT)) {
+ rpmalloc_assert(heap->full_span_count, "Heap span counter corrupted");
+ --heap->full_span_count;
+ _rpmalloc_stat_dec(&heap->span_use[0].spans_deferred);
+#if RPMALLOC_FIRST_CLASS_HEAPS
+ _rpmalloc_span_double_link_list_remove(&heap->full_span[span->size_class],
+ span);
+#endif
+ _rpmalloc_stat_dec(&heap->span_use[0].current);
+ _rpmalloc_stat_dec(&heap->size_class_use[span->size_class].spans_current);
+ if (single_span && !*single_span)
+ *single_span = span;
+ else
+ _rpmalloc_heap_cache_insert(heap, span);
+ } else {
+ if (span->size_class == SIZE_CLASS_HUGE) {
+ _rpmalloc_deallocate_huge(span);
+ } else {
+ rpmalloc_assert(span->size_class == SIZE_CLASS_LARGE,
+ "Span size class invalid");
+ rpmalloc_assert(heap->full_span_count, "Heap span counter corrupted");
+ --heap->full_span_count;
+#if RPMALLOC_FIRST_CLASS_HEAPS
+ _rpmalloc_span_double_link_list_remove(&heap->large_huge_span, span);
+#endif
+ uint32_t idx = span->span_count - 1;
+ _rpmalloc_stat_dec(&heap->span_use[idx].spans_deferred);
+ _rpmalloc_stat_dec(&heap->span_use[idx].current);
+ if (!idx && single_span && !*single_span)
+ *single_span = span;
+ else
+ _rpmalloc_heap_cache_insert(heap, span);
+ }
+ }
+ span = next_span;
+ }
+}
+
+static void _rpmalloc_heap_unmap(heap_t *heap) {
+ if (!heap->master_heap) {
+ if ((heap->finalize > 1) && !atomic_load32(&heap->child_count)) {
+ span_t *span = (span_t *)((uintptr_t)heap & _memory_span_mask);
+ _rpmalloc_span_unmap(span);
+ }
+ } else {
+ if (atomic_decr32(&heap->master_heap->child_count) == 0) {
+ _rpmalloc_heap_unmap(heap->master_heap);
+ }
+ }
+}
+
+static void _rpmalloc_heap_global_finalize(heap_t *heap) {
+ if (heap->finalize++ > 1) {
+ --heap->finalize;
+ return;
+ }
+
+ _rpmalloc_heap_finalize(heap);
+
+#if ENABLE_THREAD_CACHE
+ for (size_t iclass = 0; iclass < LARGE_CLASS_COUNT; ++iclass) {
+ span_cache_t *span_cache;
+ if (!iclass)
+ span_cache = &heap->span_cache;
+ else
+ span_cache = (span_cache_t *)(heap->span_large_cache + (iclass - 1));
+ for (size_t ispan = 0; ispan < span_cache->count; ++ispan)
+ _rpmalloc_span_unmap(span_cache->span[ispan]);
+ span_cache->count = 0;
+ }
+#endif
+
+ if (heap->full_span_count) {
+ --heap->finalize;
+ return;
+ }
+
+ for (size_t iclass = 0; iclass < SIZE_CLASS_COUNT; ++iclass) {
+ if (heap->size_class[iclass].free_list ||
+ heap->size_class[iclass].partial_span) {
+ --heap->finalize;
+ return;
+ }
+ }
+ // Heap is now completely free, unmap and remove from heap list
+ size_t list_idx = (size_t)heap->id % HEAP_ARRAY_SIZE;
+ heap_t *list_heap = _memory_heaps[list_idx];
+ if (list_heap == heap) {
+ _memory_heaps[list_idx] = heap->next_heap;
+ } else {
+ while (list_heap->next_heap != heap)
+ list_heap = list_heap->next_heap;
+ list_heap->next_heap = heap->next_heap;
+ }
+
+ _rpmalloc_heap_unmap(heap);
+}
+
+//! Insert a single span into thread heap cache, releasing to global cache if
+//! overflow
+static void _rpmalloc_heap_cache_insert(heap_t *heap, span_t *span) {
+ if (UNEXPECTED(heap->finalize != 0)) {
+ _rpmalloc_span_unmap(span);
+ _rpmalloc_heap_global_finalize(heap);
+ return;
+ }
+#if ENABLE_THREAD_CACHE
+ size_t span_count = span->span_count;
+ _rpmalloc_stat_inc(&heap->span_use[span_count - 1].spans_to_cache);
+ if (span_count == 1) {
+ span_cache_t *span_cache = &heap->span_cache;
+ span_cache->span[span_cache->count++] = span;
+ if (span_cache->count == MAX_THREAD_SPAN_CACHE) {
+ const size_t remain_count =
+ MAX_THREAD_SPAN_CACHE - THREAD_SPAN_CACHE_TRANSFER;
+#if ENABLE_GLOBAL_CACHE
+ _rpmalloc_stat_add64(&heap->thread_to_global,
+ THREAD_SPAN_CACHE_TRANSFER * _memory_span_size);
+ _rpmalloc_stat_add(&heap->span_use[span_count - 1].spans_to_global,
+ THREAD_SPAN_CACHE_TRANSFER);
+ _rpmalloc_global_cache_insert_spans(span_cache->span + remain_count,
+ span_count,
+ THREAD_SPAN_CACHE_TRANSFER);
+#else
+ for (size_t ispan = 0; ispan < THREAD_SPAN_CACHE_TRANSFER; ++ispan)
+ _rpmalloc_span_unmap(span_cache->span[remain_count + ispan]);
+#endif
+ span_cache->count = remain_count;
+ }
+ } else {
+ size_t cache_idx = span_count - 2;
+ span_large_cache_t *span_cache = heap->span_large_cache + cache_idx;
+ span_cache->span[span_cache->count++] = span;
+ const size_t cache_limit =
+ (MAX_THREAD_SPAN_LARGE_CACHE - (span_count >> 1));
+ if (span_cache->count == cache_limit) {
+ const size_t transfer_limit = 2 + (cache_limit >> 2);
+ const size_t transfer_count =
+ (THREAD_SPAN_LARGE_CACHE_TRANSFER <= transfer_limit
+ ? THREAD_SPAN_LARGE_CACHE_TRANSFER
+ : transfer_limit);
+ const size_t remain_count = cache_limit - transfer_count;
+#if ENABLE_GLOBAL_CACHE
+ _rpmalloc_stat_add64(&heap->thread_to_global,
+ transfer_count * span_count * _memory_span_size);
+ _rpmalloc_stat_add(&heap->span_use[span_count - 1].spans_to_global,
+ transfer_count);
+ _rpmalloc_global_cache_insert_spans(span_cache->span + remain_count,
+ span_count, transfer_count);
+#else
+ for (size_t ispan = 0; ispan < transfer_count; ++ispan)
+ _rpmalloc_span_unmap(span_cache->span[remain_count + ispan]);
+#endif
+ span_cache->count = remain_count;
+ }
+ }
+#else
+ (void)sizeof(heap);
+ _rpmalloc_span_unmap(span);
+#endif
+}
+
+//! Extract the given number of spans from the different cache levels
+static span_t *_rpmalloc_heap_thread_cache_extract(heap_t *heap,
+ size_t span_count) {
+ span_t *span = 0;
+#if ENABLE_THREAD_CACHE
+ span_cache_t *span_cache;
+ if (span_count == 1)
+ span_cache = &heap->span_cache;
+ else
+ span_cache = (span_cache_t *)(heap->span_large_cache + (span_count - 2));
+ if (span_cache->count) {
+ _rpmalloc_stat_inc(&heap->span_use[span_count - 1].spans_from_cache);
+ return span_cache->span[--span_cache->count];
+ }
+#endif
+ return span;
+}
+
+static span_t *_rpmalloc_heap_thread_cache_deferred_extract(heap_t *heap,
+ size_t span_count) {
+ span_t *span = 0;
+ if (span_count == 1) {
+ _rpmalloc_heap_cache_adopt_deferred(heap, &span);
+ } else {
+ _rpmalloc_heap_cache_adopt_deferred(heap, 0);
+ span = _rpmalloc_heap_thread_cache_extract(heap, span_count);
+ }
+ return span;
+}
+
+static span_t *_rpmalloc_heap_reserved_extract(heap_t *heap,
+ size_t span_count) {
+ if (heap->spans_reserved >= span_count)
+ return _rpmalloc_span_map(heap, span_count);
+ return 0;
+}
+
+//! Extract a span from the global cache
+static span_t *_rpmalloc_heap_global_cache_extract(heap_t *heap,
+ size_t span_count) {
+#if ENABLE_GLOBAL_CACHE
+#if ENABLE_THREAD_CACHE
+ span_cache_t *span_cache;
+ size_t wanted_count;
+ if (span_count == 1) {
+ span_cache = &heap->span_cache;
+ wanted_count = THREAD_SPAN_CACHE_TRANSFER;
+ } else {
+ span_cache = (span_cache_t *)(heap->span_large_cache + (span_count - 2));
+ wanted_count = THREAD_SPAN_LARGE_CACHE_TRANSFER;
+ }
+ span_cache->count = _rpmalloc_global_cache_extract_spans(
+ span_cache->span, span_count, wanted_count);
+ if (span_cache->count) {
+ _rpmalloc_stat_add64(&heap->global_to_thread,
+ span_count * span_cache->count * _memory_span_size);
+ _rpmalloc_stat_add(&heap->span_use[span_count - 1].spans_from_global,
+ span_cache->count);
+ return span_cache->span[--span_cache->count];
+ }
+#else
+ span_t *span = 0;
+ size_t count = _rpmalloc_global_cache_extract_spans(&span, span_count, 1);
+ if (count) {
+ _rpmalloc_stat_add64(&heap->global_to_thread,
+ span_count * count * _memory_span_size);
+ _rpmalloc_stat_add(&heap->span_use[span_count - 1].spans_from_global,
+ count);
+ return span;
+ }
+#endif
+#endif
+ (void)sizeof(heap);
+ (void)sizeof(span_count);
+ return 0;
+}
+
+static void _rpmalloc_inc_span_statistics(heap_t *heap, size_t span_count,
+ uint32_t class_idx) {
+ (void)sizeof(heap);
+ (void)sizeof(span_count);
+ (void)sizeof(class_idx);
+#if ENABLE_ADAPTIVE_THREAD_CACHE || ENABLE_STATISTICS
+ uint32_t idx = (uint32_t)span_count - 1;
+ uint32_t current_count =
+ (uint32_t)atomic_incr32(&heap->span_use[idx].current);
+ if (current_count > (uint32_t)atomic_load32(&heap->span_use[idx].high))
+ atomic_store32(&heap->span_use[idx].high, (int32_t)current_count);
+ _rpmalloc_stat_add_peak(&heap->size_class_use[class_idx].spans_current, 1,
+ heap->size_class_use[class_idx].spans_peak);
+#endif
+}
+
+//! Get a span from one of the cache levels (thread cache, reserved, global
+//! cache) or fallback to mapping more memory
+static span_t *
+_rpmalloc_heap_extract_new_span(heap_t *heap,
+ heap_size_class_t *heap_size_class,
+ size_t span_count, uint32_t class_idx) {
+ span_t *span;
+#if ENABLE_THREAD_CACHE
+ if (heap_size_class && heap_size_class->cache) {
+ span = heap_size_class->cache;
+ heap_size_class->cache =
+ (heap->span_cache.count
+ ? heap->span_cache.span[--heap->span_cache.count]
+ : 0);
+ _rpmalloc_inc_span_statistics(heap, span_count, class_idx);
+ return span;
+ }
+#endif
+ (void)sizeof(class_idx);
+ // Allow 50% overhead to increase cache hits
+ size_t base_span_count = span_count;
+ size_t limit_span_count =
+ (span_count > 2) ? (span_count + (span_count >> 1)) : span_count;
+ if (limit_span_count > LARGE_CLASS_COUNT)
+ limit_span_count = LARGE_CLASS_COUNT;
+ do {
+ span = _rpmalloc_heap_thread_cache_extract(heap, span_count);
+ if (EXPECTED(span != 0)) {
+ _rpmalloc_stat_inc(&heap->size_class_use[class_idx].spans_from_cache);
+ _rpmalloc_inc_span_statistics(heap, span_count, class_idx);
+ return span;
+ }
+ span = _rpmalloc_heap_thread_cache_deferred_extract(heap, span_count);
+ if (EXPECTED(span != 0)) {
+ _rpmalloc_stat_inc(&heap->size_class_use[class_idx].spans_from_cache);
+ _rpmalloc_inc_span_statistics(heap, span_count, class_idx);
+ return span;
+ }
+ span = _rpmalloc_heap_reserved_extract(heap, span_count);
+ if (EXPECTED(span != 0)) {
+ _rpmalloc_stat_inc(&heap->size_class_use[class_idx].spans_from_reserved);
+ _rpmalloc_inc_span_statistics(heap, span_count, class_idx);
+ return span;
+ }
+ span = _rpmalloc_heap_global_cache_extract(heap, span_count);
+ if (EXPECTED(span != 0)) {
+ _rpmalloc_stat_inc(&heap->size_class_use[class_idx].spans_from_cache);
+ _rpmalloc_inc_span_statistics(heap, span_count, class_idx);
+ return span;
+ }
+ ++span_count;
+ } while (span_count <= limit_span_count);
+ // Final fallback, map in more virtual memory
+ span = _rpmalloc_span_map(heap, base_span_count);
+ _rpmalloc_inc_span_statistics(heap, base_span_count, class_idx);
+ _rpmalloc_stat_inc(&heap->size_class_use[class_idx].spans_map_calls);
+ return span;
+}
+
+static void _rpmalloc_heap_initialize(heap_t *heap) {
+ memset(heap, 0, sizeof(heap_t));
+ // Get a new heap ID
+ heap->id = 1 + atomic_incr32(&_memory_heap_id);
+
+ // Link in heap in heap ID map
+ size_t list_idx = (size_t)heap->id % HEAP_ARRAY_SIZE;
+ heap->next_heap = _memory_heaps[list_idx];
+ _memory_heaps[list_idx] = heap;
+}
+
+static void _rpmalloc_heap_orphan(heap_t *heap, int first_class) {
+ heap->owner_thread = (uintptr_t)-1;
+#if RPMALLOC_FIRST_CLASS_HEAPS
+ heap_t **heap_list =
+ (first_class ? &_memory_first_class_orphan_heaps : &_memory_orphan_heaps);
+#else
+ (void)sizeof(first_class);
+ heap_t **heap_list = &_memory_orphan_heaps;
+#endif
+ heap->next_orphan = *heap_list;
+ *heap_list = heap;
+}
+
+//! Allocate a new heap from newly mapped memory pages
+static heap_t *_rpmalloc_heap_allocate_new(void) {
+ // Map in pages for a 16 heaps. If page size is greater than required size for
+ // this, map a page and use first part for heaps and remaining part for spans
+ // for allocations. Adds a lot of complexity, but saves a lot of memory on
+ // systems where page size > 64 spans (4MiB)
+ size_t heap_size = sizeof(heap_t);
+ size_t aligned_heap_size = 16 * ((heap_size + 15) / 16);
+ size_t request_heap_count = 16;
+ size_t heap_span_count = ((aligned_heap_size * request_heap_count) +
+ sizeof(span_t) + _memory_span_size - 1) /
+ _memory_span_size;
+ size_t block_size = _memory_span_size * heap_span_count;
+ size_t span_count = heap_span_count;
+ span_t *span = 0;
+ // If there are global reserved spans, use these first
+ if (_memory_global_reserve_count >= heap_span_count) {
+ span = _rpmalloc_global_get_reserved_spans(heap_span_count);
+ }
+ if (!span) {
+ if (_memory_page_size > block_size) {
+ span_count = _memory_page_size / _memory_span_size;
+ block_size = _memory_page_size;
+ // If using huge pages, make sure to grab enough heaps to avoid
+ // reallocating a huge page just to serve new heaps
+ size_t possible_heap_count =
+ (block_size - sizeof(span_t)) / aligned_heap_size;
+ if (possible_heap_count >= (request_heap_count * 16))
+ request_heap_count *= 16;
+ else if (possible_heap_count < request_heap_count)
+ request_heap_count = possible_heap_count;
+ heap_span_count = ((aligned_heap_size * request_heap_count) +
+ sizeof(span_t) + _memory_span_size - 1) /
+ _memory_span_size;
+ }
+
+ size_t align_offset = 0;
+ span = (span_t *)_rpmalloc_mmap(block_size, &align_offset);
+ if (!span)
+ return 0;
+
+ // Master span will contain the heaps
+ _rpmalloc_stat_inc(&_master_spans);
+ _rpmalloc_span_initialize(span, span_count, heap_span_count, align_offset);
+ }
+
+ size_t remain_size = _memory_span_size - sizeof(span_t);
+ heap_t *heap = (heap_t *)pointer_offset(span, sizeof(span_t));
+ _rpmalloc_heap_initialize(heap);
+
+ // Put extra heaps as orphans
+ size_t num_heaps = remain_size / aligned_heap_size;
+ if (num_heaps < request_heap_count)
+ num_heaps = request_heap_count;
+ atomic_store32(&heap->child_count, (int32_t)num_heaps - 1);
+ heap_t *extra_heap = (heap_t *)pointer_offset(heap, aligned_heap_size);
+ while (num_heaps > 1) {
+ _rpmalloc_heap_initialize(extra_heap);
+ extra_heap->master_heap = heap;
+ _rpmalloc_heap_orphan(extra_heap, 1);
+ extra_heap = (heap_t *)pointer_offset(extra_heap, aligned_heap_size);
+ --num_heaps;
+ }
+
+ if (span_count > heap_span_count) {
+ // Cap reserved spans
+ size_t remain_count = span_count - heap_span_count;
+ size_t reserve_count =
+ (remain_count > _memory_heap_reserve_count ? _memory_heap_reserve_count
+ : remain_count);
+ span_t *remain_span =
+ (span_t *)pointer_offset(span, heap_span_count * _memory_span_size);
+ _rpmalloc_heap_set_reserved_spans(heap, span, remain_span, reserve_count);
+
+ if (remain_count > reserve_count) {
+ // Set to global reserved spans
+ remain_span = (span_t *)pointer_offset(remain_span,
+ reserve_count * _memory_span_size);
+ reserve_count = remain_count - reserve_count;
+ _rpmalloc_global_set_reserved_spans(span, remain_span, reserve_count);
+ }
+ }
+
+ return heap;
+}
+
+static heap_t *_rpmalloc_heap_extract_orphan(heap_t **heap_list) {
+ heap_t *heap = *heap_list;
+ *heap_list = (heap ? heap->next_orphan : 0);
+ return heap;
+}
+
+//! Allocate a new heap, potentially reusing a previously orphaned heap
+static heap_t *_rpmalloc_heap_allocate(int first_class) {
+ heap_t *heap = 0;
+ while (!atomic_cas32_acquire(&_memory_global_lock, 1, 0))
+ _rpmalloc_spin();
+ if (first_class == 0)
+ heap = _rpmalloc_heap_extract_orphan(&_memory_orphan_heaps);
+#if RPMALLOC_FIRST_CLASS_HEAPS
+ if (!heap)
+ heap = _rpmalloc_heap_extract_orphan(&_memory_first_class_orphan_heaps);
+#endif
+ if (!heap)
+ heap = _rpmalloc_heap_allocate_new();
+ atomic_store32_release(&_memory_global_lock, 0);
+ _rpmalloc_heap_cache_adopt_deferred(heap, 0);
+ return heap;
+}
+
+static void _rpmalloc_heap_release(void *heapptr, int first_class,
+ int release_cache) {
+ heap_t *heap = (heap_t *)heapptr;
+ if (!heap)
+ return;
+ // Release thread cache spans back to global cache
+ _rpmalloc_heap_cache_adopt_deferred(heap, 0);
+ if (release_cache || heap->finalize) {
+#if ENABLE_THREAD_CACHE
+ for (size_t iclass = 0; iclass < LARGE_CLASS_COUNT; ++iclass) {
+ span_cache_t *span_cache;
+ if (!iclass)
+ span_cache = &heap->span_cache;
+ else
+ span_cache = (span_cache_t *)(heap->span_large_cache + (iclass - 1));
+ if (!span_cache->count)
+ continue;
+#if ENABLE_GLOBAL_CACHE
+ if (heap->finalize) {
+ for (size_t ispan = 0; ispan < span_cache->count; ++ispan)
+ _rpmalloc_span_unmap(span_cache->span[ispan]);
+ } else {
+ _rpmalloc_stat_add64(&heap->thread_to_global, span_cache->count *
+ (iclass + 1) *
+ _memory_span_size);
+ _rpmalloc_stat_add(&heap->span_use[iclass].spans_to_global,
+ span_cache->count);
+ _rpmalloc_global_cache_insert_spans(span_cache->span, iclass + 1,
+ span_cache->count);
+ }
+#else
+ for (size_t ispan = 0; ispan < span_cache->count; ++ispan)
+ _rpmalloc_span_unmap(span_cache->span[ispan]);
+#endif
+ span_cache->count = 0;
+ }
+#endif
+ }
+
+ if (get_thread_heap_raw() == heap)
+ set_thread_heap(0);
+
+#if ENABLE_STATISTICS
+ atomic_decr32(&_memory_active_heaps);
+ rpmalloc_assert(atomic_load32(&_memory_active_heaps) >= 0,
+ "Still active heaps during finalization");
+#endif
+
+ // If we are forcibly terminating with _exit the state of the
+ // lock atomic is unknown and it's best to just go ahead and exit
+ if (get_thread_id() != _rpmalloc_main_thread_id) {
+ while (!atomic_cas32_acquire(&_memory_global_lock, 1, 0))
+ _rpmalloc_spin();
+ }
+ _rpmalloc_heap_orphan(heap, first_class);
+ atomic_store32_release(&_memory_global_lock, 0);
+}
+
+static void _rpmalloc_heap_release_raw(void *heapptr, int release_cache) {
+ _rpmalloc_heap_release(heapptr, 0, release_cache);
+}
+
+static void _rpmalloc_heap_release_raw_fc(void *heapptr) {
+ _rpmalloc_heap_release_raw(heapptr, 1);
+}
+
+static void _rpmalloc_heap_finalize(heap_t *heap) {
+ if (heap->spans_reserved) {
+ span_t *span = _rpmalloc_span_map(heap, heap->spans_reserved);
+ _rpmalloc_span_unmap(span);
+ heap->spans_reserved = 0;
+ }
+
+ _rpmalloc_heap_cache_adopt_deferred(heap, 0);
+
+ for (size_t iclass = 0; iclass < SIZE_CLASS_COUNT; ++iclass) {
+ if (heap->size_class[iclass].cache)
+ _rpmalloc_span_unmap(heap->size_class[iclass].cache);
+ heap->size_class[iclass].cache = 0;
+ span_t *span = heap->size_class[iclass].partial_span;
+ while (span) {
+ span_t *next = span->next;
+ _rpmalloc_span_finalize(heap, iclass, span,
+ &heap->size_class[iclass].partial_span);
+ span = next;
+ }
+ // If class still has a free list it must be a full span
+ if (heap->size_class[iclass].free_list) {
+ span_t *class_span =
+ (span_t *)((uintptr_t)heap->size_class[iclass].free_list &
+ _memory_span_mask);
+ span_t **list = 0;
+#if RPMALLOC_FIRST_CLASS_HEAPS
+ list = &heap->full_span[iclass];
+#endif
+ --heap->full_span_count;
+ if (!_rpmalloc_span_finalize(heap, iclass, class_span, list)) {
+ if (list)
+ _rpmalloc_span_double_link_list_remove(list, class_span);
+ _rpmalloc_span_double_link_list_add(
+ &heap->size_class[iclass].partial_span, class_span);
+ }
+ }
+ }
+
+#if ENABLE_THREAD_CACHE
+ for (size_t iclass = 0; iclass < LARGE_CLASS_COUNT; ++iclass) {
+ span_cache_t *span_cache;
+ if (!iclass)
+ span_cache = &heap->span_cache;
+ else
+ span_cache = (span_cache_t *)(heap->span_large_cache + (iclass - 1));
+ for (size_t ispan = 0; ispan < span_cache->count; ++ispan)
+ _rpmalloc_span_unmap(span_cache->span[ispan]);
+ span_cache->count = 0;
+ }
+#endif
+ rpmalloc_assert(!atomic_load_ptr(&heap->span_free_deferred),
+ "Heaps still active during finalization");
+}
+
+////////////
+///
+/// Allocation entry points
+///
+//////
+
+//! Pop first block from a free list
+static void *free_list_pop(void **list) {
+ void *block = *list;
+ *list = *((void **)block);
+ return block;
+}
+
+//! Allocate a small/medium sized memory block from the given heap
+static void *_rpmalloc_allocate_from_heap_fallback(
+ heap_t *heap, heap_size_class_t *heap_size_class, uint32_t class_idx) {
+ span_t *span = heap_size_class->partial_span;
+ if (EXPECTED(span != 0)) {
+ rpmalloc_assert(span->block_count ==
+ _memory_size_class[span->size_class].block_count,
+ "Span block count corrupted");
+ rpmalloc_assert(!_rpmalloc_span_is_fully_utilized(span),
+ "Internal failure");
+ void *block;
+ if (span->free_list) {
+ // Span local free list is not empty, swap to size class free list
+ block = free_list_pop(&span->free_list);
+ heap_size_class->free_list = span->free_list;
+ span->free_list = 0;
+ } else {
+ // If the span did not fully initialize free list, link up another page
+ // worth of blocks
+ void *block_start = pointer_offset(
+ span, SPAN_HEADER_SIZE +
+ ((size_t)span->free_list_limit * span->block_size));
+ span->free_list_limit += free_list_partial_init(
+ &heap_size_class->free_list, &block,
+ (void *)((uintptr_t)block_start & ~(_memory_page_size - 1)),
+ block_start, span->block_count - span->free_list_limit,
+ span->block_size);
+ }
+ rpmalloc_assert(span->free_list_limit <= span->block_count,
+ "Span block count corrupted");
+ span->used_count = span->free_list_limit;
+
+ // Swap in deferred free list if present
+ if (atomic_load_ptr(&span->free_list_deferred))
+ _rpmalloc_span_extract_free_list_deferred(span);
+
+ // If span is still not fully utilized keep it in partial list and early
+ // return block
+ if (!_rpmalloc_span_is_fully_utilized(span))
+ return block;
+
+ // The span is fully utilized, unlink from partial list and add to fully
+ // utilized list
+ _rpmalloc_span_double_link_list_pop_head(&heap_size_class->partial_span,
+ span);
+#if RPMALLOC_FIRST_CLASS_HEAPS
+ _rpmalloc_span_double_link_list_add(&heap->full_span[class_idx], span);
+#endif
+ ++heap->full_span_count;
+ return block;
+ }
+
+ // Find a span in one of the cache levels
+ span = _rpmalloc_heap_extract_new_span(heap, heap_size_class, 1, class_idx);
+ if (EXPECTED(span != 0)) {
+ // Mark span as owned by this heap and set base data, return first block
+ return _rpmalloc_span_initialize_new(heap, heap_size_class, span,
+ class_idx);
+ }
+
+ return 0;
+}
+
+//! Allocate a small sized memory block from the given heap
+static void *_rpmalloc_allocate_small(heap_t *heap, size_t size) {
+ rpmalloc_assert(heap, "No thread heap");
+ // Small sizes have unique size classes
+ const uint32_t class_idx =
+ (uint32_t)((size + (SMALL_GRANULARITY - 1)) >> SMALL_GRANULARITY_SHIFT);
+ heap_size_class_t *heap_size_class = heap->size_class + class_idx;
+ _rpmalloc_stat_inc_alloc(heap, class_idx);
+ if (EXPECTED(heap_size_class->free_list != 0))
+ return free_list_pop(&heap_size_class->free_list);
+ return _rpmalloc_allocate_from_heap_fallback(heap, heap_size_class,
+ class_idx);
+}
+
+//! Allocate a medium sized memory block from the given heap
+static void *_rpmalloc_allocate_medium(heap_t *heap, size_t size) {
+ rpmalloc_assert(heap, "No thread heap");
+ // Calculate the size class index and do a dependent lookup of the final class
+ // index (in case of merged classes)
+ const uint32_t base_idx =
+ (uint32_t)(SMALL_CLASS_COUNT +
+ ((size - (SMALL_SIZE_LIMIT + 1)) >> MEDIUM_GRANULARITY_SHIFT));
+ const uint32_t class_idx = _memory_size_class[base_idx].class_idx;
+ heap_size_class_t *heap_size_class = heap->size_class + class_idx;
+ _rpmalloc_stat_inc_alloc(heap, class_idx);
+ if (EXPECTED(heap_size_class->free_list != 0))
+ return free_list_pop(&heap_size_class->free_list);
+ return _rpmalloc_allocate_from_heap_fallback(heap, heap_size_class,
+ class_idx);
+}
+
+//! Allocate a large sized memory block from the given heap
+static void *_rpmalloc_allocate_large(heap_t *heap, size_t size) {
+ rpmalloc_assert(heap, "No thread heap");
+ // Calculate number of needed max sized spans (including header)
+ // Since this function is never called if size > LARGE_SIZE_LIMIT
+ // the span_count is guaranteed to be <= LARGE_CLASS_COUNT
+ size += SPAN_HEADER_SIZE;
+ size_t span_count = size >> _memory_span_size_shift;
+ if (size & (_memory_span_size - 1))
+ ++span_count;
+
+ // Find a span in one of the cache levels
+ span_t *span =
+ _rpmalloc_heap_extract_new_span(heap, 0, span_count, SIZE_CLASS_LARGE);
+ if (!span)
+ return span;
+
+ // Mark span as owned by this heap and set base data
+ rpmalloc_assert(span->span_count >= span_count, "Internal failure");
+ span->size_class = SIZE_CLASS_LARGE;
+ span->heap = heap;
+
+#if RPMALLOC_FIRST_CLASS_HEAPS
+ _rpmalloc_span_double_link_list_add(&heap->large_huge_span, span);
+#endif
+ ++heap->full_span_count;
+
+ return pointer_offset(span, SPAN_HEADER_SIZE);
+}
+
+//! Allocate a huge block by mapping memory pages directly
+static void *_rpmalloc_allocate_huge(heap_t *heap, size_t size) {
+ rpmalloc_assert(heap, "No thread heap");
+ _rpmalloc_heap_cache_adopt_deferred(heap, 0);
+ size += SPAN_HEADER_SIZE;
+ size_t num_pages = size >> _memory_page_size_shift;
+ if (size & (_memory_page_size - 1))
+ ++num_pages;
+ size_t align_offset = 0;
+ span_t *span =
+ (span_t *)_rpmalloc_mmap(num_pages * _memory_page_size, &align_offset);
+ if (!span)
+ return span;
+
+ // Store page count in span_count
+ span->size_class = SIZE_CLASS_HUGE;
+ span->span_count = (uint32_t)num_pages;
+ span->align_offset = (uint32_t)align_offset;
+ span->heap = heap;
+ _rpmalloc_stat_add_peak(&_huge_pages_current, num_pages, _huge_pages_peak);
+
+#if RPMALLOC_FIRST_CLASS_HEAPS
+ _rpmalloc_span_double_link_list_add(&heap->large_huge_span, span);
+#endif
+ ++heap->full_span_count;
+
+ return pointer_offset(span, SPAN_HEADER_SIZE);
+}
+
+//! Allocate a block of the given size
+static void *_rpmalloc_allocate(heap_t *heap, size_t size) {
+ _rpmalloc_stat_add64(&_allocation_counter, 1);
+ if (EXPECTED(size <= SMALL_SIZE_LIMIT))
+ return _rpmalloc_allocate_small(heap, size);
+ else if (size <= _memory_medium_size_limit)
+ return _rpmalloc_allocate_medium(heap, size);
+ else if (size <= LARGE_SIZE_LIMIT)
+ return _rpmalloc_allocate_large(heap, size);
+ return _rpmalloc_allocate_huge(heap, size);
+}
+
+static void *_rpmalloc_aligned_allocate(heap_t *heap, size_t alignment,
+ size_t size) {
+ if (alignment <= SMALL_GRANULARITY)
+ return _rpmalloc_allocate(heap, size);
+
+#if ENABLE_VALIDATE_ARGS
+ if ((size + alignment) < size) {
+ errno = EINVAL;
+ return 0;
+ }
+ if (alignment & (alignment - 1)) {
+ errno = EINVAL;
+ return 0;
+ }
+#endif
+
+ if ((alignment <= SPAN_HEADER_SIZE) && (size < _memory_medium_size_limit)) {
+ // If alignment is less or equal to span header size (which is power of
+ // two), and size aligned to span header size multiples is less than size +
+ // alignment, then use natural alignment of blocks to provide alignment
+ size_t multiple_size = size ? (size + (SPAN_HEADER_SIZE - 1)) &
+ ~(uintptr_t)(SPAN_HEADER_SIZE - 1)
+ : SPAN_HEADER_SIZE;
+ rpmalloc_assert(!(multiple_size % SPAN_HEADER_SIZE),
+ "Failed alignment calculation");
+ if (multiple_size <= (size + alignment))
+ return _rpmalloc_allocate(heap, multiple_size);
+ }
+
+ void *ptr = 0;
+ size_t align_mask = alignment - 1;
+ if (alignment <= _memory_page_size) {
+ ptr = _rpmalloc_allocate(heap, size + alignment);
+ if ((uintptr_t)ptr & align_mask) {
+ ptr = (void *)(((uintptr_t)ptr & ~(uintptr_t)align_mask) + alignment);
+ // Mark as having aligned blocks
+ span_t *span = (span_t *)((uintptr_t)ptr & _memory_span_mask);
+ span->flags |= SPAN_FLAG_ALIGNED_BLOCKS;
+ }
+ return ptr;
+ }
+
+ // Fallback to mapping new pages for this request. Since pointers passed
+ // to rpfree must be able to reach the start of the span by bitmasking of
+ // the address with the span size, the returned aligned pointer from this
+ // function must be with a span size of the start of the mapped area.
+ // In worst case this requires us to loop and map pages until we get a
+ // suitable memory address. It also means we can never align to span size
+ // or greater, since the span header will push alignment more than one
+ // span size away from span start (thus causing pointer mask to give us
+ // an invalid span start on free)
+ if (alignment & align_mask) {
+ errno = EINVAL;
+ return 0;
+ }
+ if (alignment >= _memory_span_size) {
+ errno = EINVAL;
+ return 0;
+ }
+
+ size_t extra_pages = alignment / _memory_page_size;
+
+ // Since each span has a header, we will at least need one extra memory page
+ size_t num_pages = 1 + (size / _memory_page_size);
+ if (size & (_memory_page_size - 1))
+ ++num_pages;
+
+ if (extra_pages > num_pages)
+ num_pages = 1 + extra_pages;
+
+ size_t original_pages = num_pages;
+ size_t limit_pages = (_memory_span_size / _memory_page_size) * 2;
+ if (limit_pages < (original_pages * 2))
+ limit_pages = original_pages * 2;
+
+ size_t mapped_size, align_offset;
+ span_t *span;
+
+retry:
+ align_offset = 0;
+ mapped_size = num_pages * _memory_page_size;
+
+ span = (span_t *)_rpmalloc_mmap(mapped_size, &align_offset);
+ if (!span) {
+ errno = ENOMEM;
+ return 0;
+ }
+ ptr = pointer_offset(span, SPAN_HEADER_SIZE);
+
+ if ((uintptr_t)ptr & align_mask)
+ ptr = (void *)(((uintptr_t)ptr & ~(uintptr_t)align_mask) + alignment);
+
+ if (((size_t)pointer_diff(ptr, span) >= _memory_span_size) ||
+ (pointer_offset(ptr, size) > pointer_offset(span, mapped_size)) ||
+ (((uintptr_t)ptr & _memory_span_mask) != (uintptr_t)span)) {
+ _rpmalloc_unmap(span, mapped_size, align_offset, mapped_size);
+ ++num_pages;
+ if (num_pages > limit_pages) {
+ errno = EINVAL;
+ return 0;
+ }
+ goto retry;
+ }
+
+ // Store page count in span_count
+ span->size_class = SIZE_CLASS_HUGE;
+ span->span_count = (uint32_t)num_pages;
+ span->align_offset = (uint32_t)align_offset;
+ span->heap = heap;
+ _rpmalloc_stat_add_peak(&_huge_pages_current, num_pages, _huge_pages_peak);
+
+#if RPMALLOC_FIRST_CLASS_HEAPS
+ _rpmalloc_span_double_link_list_add(&heap->large_huge_span, span);
+#endif
+ ++heap->full_span_count;
+
+ _rpmalloc_stat_add64(&_allocation_counter, 1);
+
+ return ptr;
+}
+
+////////////
+///
+/// Deallocation entry points
+///
+//////
+
+//! Deallocate the given small/medium memory block in the current thread local
+//! heap
+static void _rpmalloc_deallocate_direct_small_or_medium(span_t *span,
+ void *block) {
+ heap_t *heap = span->heap;
+ rpmalloc_assert(heap->owner_thread == get_thread_id() ||
+ !heap->owner_thread || heap->finalize,
+ "Internal failure");
+ // Add block to free list
+ if (UNEXPECTED(_rpmalloc_span_is_fully_utilized(span))) {
+ span->used_count = span->block_count;
+#if RPMALLOC_FIRST_CLASS_HEAPS
+ _rpmalloc_span_double_link_list_remove(&heap->full_span[span->size_class],
+ span);
+#endif
+ _rpmalloc_span_double_link_list_add(
+ &heap->size_class[span->size_class].partial_span, span);
+ --heap->full_span_count;
+ }
+ *((void **)block) = span->free_list;
+ --span->used_count;
+ span->free_list = block;
+ if (UNEXPECTED(span->used_count == span->list_size)) {
+ // If there are no used blocks it is guaranteed that no other external
+ // thread is accessing the span
+ if (span->used_count) {
+ // Make sure we have synchronized the deferred list and list size by using
+ // acquire semantics and guarantee that no external thread is accessing
+ // span concurrently
+ void *free_list;
+ do {
+ free_list = atomic_exchange_ptr_acquire(&span->free_list_deferred,
+ INVALID_POINTER);
+ } while (free_list == INVALID_POINTER);
+ atomic_store_ptr_release(&span->free_list_deferred, free_list);
+ }
+ _rpmalloc_span_double_link_list_remove(
+ &heap->size_class[span->size_class].partial_span, span);
+ _rpmalloc_span_release_to_cache(heap, span);
+ }
+}
+
+static void _rpmalloc_deallocate_defer_free_span(heap_t *heap, span_t *span) {
+ if (span->size_class != SIZE_CLASS_HUGE)
+ _rpmalloc_stat_inc(&heap->span_use[span->span_count - 1].spans_deferred);
+ // This list does not need ABA protection, no mutable side state
+ do {
+ span->free_list = (void *)atomic_load_ptr(&heap->span_free_deferred);
+ } while (!atomic_cas_ptr(&heap->span_free_deferred, span, span->free_list));
+}
+
+//! Put the block in the deferred free list of the owning span
+static void _rpmalloc_deallocate_defer_small_or_medium(span_t *span,
+ void *block) {
+ // The memory ordering here is a bit tricky, to avoid having to ABA protect
+ // the deferred free list to avoid desynchronization of list and list size
+ // we need to have acquire semantics on successful CAS of the pointer to
+ // guarantee the list_size variable validity + release semantics on pointer
+ // store
+ void *free_list;
+ do {
+ free_list =
+ atomic_exchange_ptr_acquire(&span->free_list_deferred, INVALID_POINTER);
+ } while (free_list == INVALID_POINTER);
+ *((void **)block) = free_list;
+ uint32_t free_count = ++span->list_size;
+ int all_deferred_free = (free_count == span->block_count);
+ atomic_store_ptr_release(&span->free_list_deferred, block);
+ if (all_deferred_free) {
+ // Span was completely freed by this block. Due to the INVALID_POINTER spin
+ // lock no other thread can reach this state simultaneously on this span.
+ // Safe to move to owner heap deferred cache
+ _rpmalloc_deallocate_defer_free_span(span->heap, span);
+ }
+}
+
+static void _rpmalloc_deallocate_small_or_medium(span_t *span, void *p) {
+ _rpmalloc_stat_inc_free(span->heap, span->size_class);
+ if (span->flags & SPAN_FLAG_ALIGNED_BLOCKS) {
+ // Realign pointer to block start
+ void *blocks_start = pointer_offset(span, SPAN_HEADER_SIZE);
+ uint32_t block_offset = (uint32_t)pointer_diff(p, blocks_start);
+ p = pointer_offset(p, -(int32_t)(block_offset % span->block_size));
+ }
+ // Check if block belongs to this heap or if deallocation should be deferred
+#if RPMALLOC_FIRST_CLASS_HEAPS
+ int defer =
+ (span->heap->owner_thread &&
+ (span->heap->owner_thread != get_thread_id()) && !span->heap->finalize);
+#else
+ int defer =
+ ((span->heap->owner_thread != get_thread_id()) && !span->heap->finalize);
+#endif
+ if (!defer)
+ _rpmalloc_deallocate_direct_small_or_medium(span, p);
+ else
+ _rpmalloc_deallocate_defer_small_or_medium(span, p);
+}
+
+//! Deallocate the given large memory block to the current heap
+static void _rpmalloc_deallocate_large(span_t *span) {
+ rpmalloc_assert(span->size_class == SIZE_CLASS_LARGE, "Bad span size class");
+ rpmalloc_assert(!(span->flags & SPAN_FLAG_MASTER) ||
+ !(span->flags & SPAN_FLAG_SUBSPAN),
+ "Span flag corrupted");
+ rpmalloc_assert((span->flags & SPAN_FLAG_MASTER) ||
+ (span->flags & SPAN_FLAG_SUBSPAN),
+ "Span flag corrupted");
+ // We must always defer (unless finalizing) if from another heap since we
+ // cannot touch the list or counters of another heap
+#if RPMALLOC_FIRST_CLASS_HEAPS
+ int defer =
+ (span->heap->owner_thread &&
+ (span->heap->owner_thread != get_thread_id()) && !span->heap->finalize);
+#else
+ int defer =
+ ((span->heap->owner_thread != get_thread_id()) && !span->heap->finalize);
+#endif
+ if (defer) {
+ _rpmalloc_deallocate_defer_free_span(span->heap, span);
+ return;
+ }
+ rpmalloc_assert(span->heap->full_span_count, "Heap span counter corrupted");
+ --span->heap->full_span_count;
+#if RPMALLOC_FIRST_CLASS_HEAPS
+ _rpmalloc_span_double_link_list_remove(&span->heap->large_huge_span, span);
+#endif
+#if ENABLE_ADAPTIVE_THREAD_CACHE || ENABLE_STATISTICS
+ // Decrease counter
+ size_t idx = span->span_count - 1;
+ atomic_decr32(&span->heap->span_use[idx].current);
+#endif
+ heap_t *heap = span->heap;
+ rpmalloc_assert(heap, "No thread heap");
+#if ENABLE_THREAD_CACHE
+ const int set_as_reserved =
+ ((span->span_count > 1) && (heap->span_cache.count == 0) &&
+ !heap->finalize && !heap->spans_reserved);
+#else
+ const int set_as_reserved =
+ ((span->span_count > 1) && !heap->finalize && !heap->spans_reserved);
+#endif
+ if (set_as_reserved) {
+ heap->span_reserve = span;
+ heap->spans_reserved = span->span_count;
+ if (span->flags & SPAN_FLAG_MASTER) {
+ heap->span_reserve_master = span;
+ } else { // SPAN_FLAG_SUBSPAN
+ span_t *master = (span_t *)pointer_offset(
+ span,
+ -(intptr_t)((size_t)span->offset_from_master * _memory_span_size));
+ heap->span_reserve_master = master;
+ rpmalloc_assert(master->flags & SPAN_FLAG_MASTER, "Span flag corrupted");
+ rpmalloc_assert(atomic_load32(&master->remaining_spans) >=
+ (int32_t)span->span_count,
+ "Master span count corrupted");
+ }
+ _rpmalloc_stat_inc(&heap->span_use[idx].spans_to_reserved);
+ } else {
+ // Insert into cache list
+ _rpmalloc_heap_cache_insert(heap, span);
+ }
+}
+
+//! Deallocate the given huge span
+static void _rpmalloc_deallocate_huge(span_t *span) {
+ rpmalloc_assert(span->heap, "No span heap");
+#if RPMALLOC_FIRST_CLASS_HEAPS
+ int defer =
+ (span->heap->owner_thread &&
+ (span->heap->owner_thread != get_thread_id()) && !span->heap->finalize);
+#else
+ int defer =
+ ((span->heap->owner_thread != get_thread_id()) && !span->heap->finalize);
+#endif
+ if (defer) {
+ _rpmalloc_deallocate_defer_free_span(span->heap, span);
+ return;
+ }
+ rpmalloc_assert(span->heap->full_span_count, "Heap span counter corrupted");
+ --span->heap->full_span_count;
+#if RPMALLOC_FIRST_CLASS_HEAPS
+ _rpmalloc_span_double_link_list_remove(&span->heap->large_huge_span, span);
+#endif
+
+ // Oversized allocation, page count is stored in span_count
+ size_t num_pages = span->span_count;
+ _rpmalloc_unmap(span, num_pages * _memory_page_size, span->align_offset,
+ num_pages * _memory_page_size);
+ _rpmalloc_stat_sub(&_huge_pages_current, num_pages);
+}
+
+//! Deallocate the given block
+static void _rpmalloc_deallocate(void *p) {
+ _rpmalloc_stat_add64(&_deallocation_counter, 1);
+ // Grab the span (always at start of span, using span alignment)
+ span_t *span = (span_t *)((uintptr_t)p & _memory_span_mask);
+ if (UNEXPECTED(!span))
+ return;
+ if (EXPECTED(span->size_class < SIZE_CLASS_COUNT))
+ _rpmalloc_deallocate_small_or_medium(span, p);
+ else if (span->size_class == SIZE_CLASS_LARGE)
+ _rpmalloc_deallocate_large(span);
+ else
+ _rpmalloc_deallocate_huge(span);
+}
+
+////////////
+///
+/// Reallocation entry points
+///
+//////
+
+static size_t _rpmalloc_usable_size(void *p);
+
+//! Reallocate the given block to the given size
+static void *_rpmalloc_reallocate(heap_t *heap, void *p, size_t size,
+ size_t oldsize, unsigned int flags) {
+ if (p) {
+ // Grab the span using guaranteed span alignment
+ span_t *span = (span_t *)((uintptr_t)p & _memory_span_mask);
+ if (EXPECTED(span->size_class < SIZE_CLASS_COUNT)) {
+ // Small/medium sized block
+ rpmalloc_assert(span->span_count == 1, "Span counter corrupted");
+ void *blocks_start = pointer_offset(span, SPAN_HEADER_SIZE);
+ uint32_t block_offset = (uint32_t)pointer_diff(p, blocks_start);
+ uint32_t block_idx = block_offset / span->block_size;
+ void *block =
+ pointer_offset(blocks_start, (size_t)block_idx * span->block_size);
+ if (!oldsize)
+ oldsize =
+ (size_t)((ptrdiff_t)span->block_size - pointer_diff(p, block));
+ if ((size_t)span->block_size >= size) {
+ // Still fits in block, never mind trying to save memory, but preserve
+ // data if alignment changed
+ if ((p != block) && !(flags & RPMALLOC_NO_PRESERVE))
+ memmove(block, p, oldsize);
+ return block;
+ }
+ } else if (span->size_class == SIZE_CLASS_LARGE) {
+ // Large block
+ size_t total_size = size + SPAN_HEADER_SIZE;
+ size_t num_spans = total_size >> _memory_span_size_shift;
+ if (total_size & (_memory_span_mask - 1))
+ ++num_spans;
+ size_t current_spans = span->span_count;
+ void *block = pointer_offset(span, SPAN_HEADER_SIZE);
+ if (!oldsize)
+ oldsize = (current_spans * _memory_span_size) -
+ (size_t)pointer_diff(p, block) - SPAN_HEADER_SIZE;
+ if ((current_spans >= num_spans) && (total_size >= (oldsize / 2))) {
+ // Still fits in block, never mind trying to save memory, but preserve
+ // data if alignment changed
+ if ((p != block) && !(flags & RPMALLOC_NO_PRESERVE))
+ memmove(block, p, oldsize);
+ return block;
+ }
+ } else {
+ // Oversized block
+ size_t total_size = size + SPAN_HEADER_SIZE;
+ size_t num_pages = total_size >> _memory_page_size_shift;
+ if (total_size & (_memory_page_size - 1))
+ ++num_pages;
+ // Page count is stored in span_count
+ size_t current_pages = span->span_count;
+ void *block = pointer_offset(span, SPAN_HEADER_SIZE);
+ if (!oldsize)
+ oldsize = (current_pages * _memory_page_size) -
+ (size_t)pointer_diff(p, block) - SPAN_HEADER_SIZE;
+ if ((current_pages >= num_pages) && (num_pages >= (current_pages / 2))) {
+ // Still fits in block, never mind trying to save memory, but preserve
+ // data if alignment changed
+ if ((p != block) && !(flags & RPMALLOC_NO_PRESERVE))
+ memmove(block, p, oldsize);
+ return block;
+ }
+ }
+ } else {
+ oldsize = 0;
+ }
+
+ if (!!(flags & RPMALLOC_GROW_OR_FAIL))
+ return 0;
+
+ // Size is greater than block size, need to allocate a new block and
+ // deallocate the old Avoid hysteresis by overallocating if increase is small
+ // (below 37%)
+ size_t lower_bound = oldsize + (oldsize >> 2) + (oldsize >> 3);
+ size_t new_size =
+ (size > lower_bound) ? size : ((size > oldsize) ? lower_bound : size);
+ void *block = _rpmalloc_allocate(heap, new_size);
+ if (p && block) {
+ if (!(flags & RPMALLOC_NO_PRESERVE))
+ memcpy(block, p, oldsize < new_size ? oldsize : new_size);
+ _rpmalloc_deallocate(p);
+ }
+
+ return block;
+}
+
+static void *_rpmalloc_aligned_reallocate(heap_t *heap, void *ptr,
+ size_t alignment, size_t size,
+ size_t oldsize, unsigned int flags) {
+ if (alignment <= SMALL_GRANULARITY)
+ return _rpmalloc_reallocate(heap, ptr, size, oldsize, flags);
+
+ int no_alloc = !!(flags & RPMALLOC_GROW_OR_FAIL);
+ size_t usablesize = (ptr ? _rpmalloc_usable_size(ptr) : 0);
+ if ((usablesize >= size) && !((uintptr_t)ptr & (alignment - 1))) {
+ if (no_alloc || (size >= (usablesize / 2)))
+ return ptr;
+ }
+ // Aligned alloc marks span as having aligned blocks
+ void *block =
+ (!no_alloc ? _rpmalloc_aligned_allocate(heap, alignment, size) : 0);
+ if (EXPECTED(block != 0)) {
+ if (!(flags & RPMALLOC_NO_PRESERVE) && ptr) {
+ if (!oldsize)
+ oldsize = usablesize;
+ memcpy(block, ptr, oldsize < size ? oldsize : size);
+ }
+ _rpmalloc_deallocate(ptr);
+ }
+ return block;
+}
+
+////////////
+///
+/// Initialization, finalization and utility
+///
+//////
+
+//! Get the usable size of the given block
+static size_t _rpmalloc_usable_size(void *p) {
+ // Grab the span using guaranteed span alignment
+ span_t *span = (span_t *)((uintptr_t)p & _memory_span_mask);
+ if (span->size_class < SIZE_CLASS_COUNT) {
+ // Small/medium block
+ void *blocks_start = pointer_offset(span, SPAN_HEADER_SIZE);
+ return span->block_size -
+ ((size_t)pointer_diff(p, blocks_start) % span->block_size);
+ }
+ if (span->size_class == SIZE_CLASS_LARGE) {
+ // Large block
+ size_t current_spans = span->span_count;
+ return (current_spans * _memory_span_size) - (size_t)pointer_diff(p, span);
+ }
+ // Oversized block, page count is stored in span_count
+ size_t current_pages = span->span_count;
+ return (current_pages * _memory_page_size) - (size_t)pointer_diff(p, span);
+}
+
+//! Adjust and optimize the size class properties for the given class
+static void _rpmalloc_adjust_size_class(size_t iclass) {
+ size_t block_size = _memory_size_class[iclass].block_size;
+ size_t block_count = (_memory_span_size - SPAN_HEADER_SIZE) / block_size;
+
+ _memory_size_class[iclass].block_count = (uint16_t)block_count;
+ _memory_size_class[iclass].class_idx = (uint16_t)iclass;
+
+ // Check if previous size classes can be merged
+ if (iclass >= SMALL_CLASS_COUNT) {
+ size_t prevclass = iclass;
+ while (prevclass > 0) {
+ --prevclass;
+ // A class can be merged if number of pages and number of blocks are equal
+ if (_memory_size_class[prevclass].block_count ==
+ _memory_size_class[iclass].block_count)
+ memcpy(_memory_size_class + prevclass, _memory_size_class + iclass,
+ sizeof(_memory_size_class[iclass]));
+ else
+ break;
+ }
+ }
+}
+
+//! Initialize the allocator and setup global data
+extern inline int rpmalloc_initialize(void) {
+ if (_rpmalloc_initialized) {
+ rpmalloc_thread_initialize();
+ return 0;
+ }
+ return rpmalloc_initialize_config(0);
+}
+
+int rpmalloc_initialize_config(const rpmalloc_config_t *config) {
+ if (_rpmalloc_initialized) {
+ rpmalloc_thread_initialize();
+ return 0;
+ }
+ _rpmalloc_initialized = 1;
+
+ if (config)
+ memcpy(&_memory_config, config, sizeof(rpmalloc_config_t));
+ else
+ memset(&_memory_config, 0, sizeof(rpmalloc_config_t));
+
+ if (!_memory_config.memory_map || !_memory_config.memory_unmap) {
+ _memory_config.memory_map = _rpmalloc_mmap_os;
+ _memory_config.memory_unmap = _rpmalloc_unmap_os;
+ }
+
+#if PLATFORM_WINDOWS
+ SYSTEM_INFO system_info;
+ memset(&system_info, 0, sizeof(system_info));
+ GetSystemInfo(&system_info);
+ _memory_map_granularity = system_info.dwAllocationGranularity;
+#else
+ _memory_map_granularity = (size_t)sysconf(_SC_PAGESIZE);
+#endif
+
+#if RPMALLOC_CONFIGURABLE
+ _memory_page_size = _memory_config.page_size;
+#else
+ _memory_page_size = 0;
+#endif
+ _memory_huge_pages = 0;
+ if (!_memory_page_size) {
+#if PLATFORM_WINDOWS
+ _memory_page_size = system_info.dwPageSize;
+#else
+ _memory_page_size = _memory_map_granularity;
+ if (_memory_config.enable_huge_pages) {
+#if defined(__linux__)
+ size_t huge_page_size = 0;
+ FILE *meminfo = fopen("/proc/meminfo", "r");
+ if (meminfo) {
+ char line[128];
+ while (!huge_page_size && fgets(line, sizeof(line) - 1, meminfo)) {
+ line[sizeof(line) - 1] = 0;
+ if (strstr(line, "Hugepagesize:"))
+ huge_page_size = (size_t)strtol(line + 13, 0, 10) * 1024;
+ }
+ fclose(meminfo);
+ }
+ if (huge_page_size) {
+ _memory_huge_pages = 1;
+ _memory_page_size = huge_page_size;
+ _memory_map_granularity = huge_page_size;
+ }
+#elif defined(__FreeBSD__)
+ int rc;
+ size_t sz = sizeof(rc);
+
+ if (sysctlbyname("vm.pmap.pg_ps_enabled", &rc, &sz, NULL, 0) == 0 &&
+ rc == 1) {
+ _memory_huge_pages = 1;
+ _memory_page_size = 2 * 1024 * 1024;
+ _memory_map_granularity = _memory_page_size;
+ }
+#elif defined(__APPLE__) || defined(__NetBSD__)
+ _memory_huge_pages = 1;
+ _memory_page_size = 2 * 1024 * 1024;
+ _memory_map_granularity = _memory_page_size;
+#endif
+ }
+#endif
+ } else {
+ if (_memory_config.enable_huge_pages)
+ _memory_huge_pages = 1;
+ }
+
+#if PLATFORM_WINDOWS
+ if (_memory_config.enable_huge_pages) {
+ HANDLE token = 0;
+ size_t large_page_minimum = GetLargePageMinimum();
+ if (large_page_minimum)
+ OpenProcessToken(GetCurrentProcess(),
+ TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &token);
+ if (token) {
+ LUID luid;
+ if (LookupPrivilegeValue(0, SE_LOCK_MEMORY_NAME, &luid)) {
+ TOKEN_PRIVILEGES token_privileges;
+ memset(&token_privileges, 0, sizeof(token_privileges));
+ token_privileges.PrivilegeCount = 1;
+ token_privileges.Privileges[0].Luid = luid;
+ token_privileges.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
+ if (AdjustTokenPrivileges(token, FALSE, &token_privileges, 0, 0, 0)) {
+ if (GetLastError() == ERROR_SUCCESS)
+ _memory_huge_pages = 1;
+ }
+ }
+ CloseHandle(token);
+ }
+ if (_memory_huge_pages) {
+ if (large_page_minimum > _memory_page_size)
+ _memory_page_size = large_page_minimum;
+ if (large_page_minimum > _memory_map_granularity)
+ _memory_map_granularity = large_page_minimum;
+ }
+ }
+#endif
+
+ size_t min_span_size = 256;
+ size_t max_page_size;
+#if UINTPTR_MAX > 0xFFFFFFFF
+ max_page_size = 4096ULL * 1024ULL * 1024ULL;
+#else
+ max_page_size = 4 * 1024 * 1024;
+#endif
+ if (_memory_page_size < min_span_size)
+ _memory_page_size = min_span_size;
+ if (_memory_page_size > max_page_size)
+ _memory_page_size = max_page_size;
+ _memory_page_size_shift = 0;
+ size_t page_size_bit = _memory_page_size;
+ while (page_size_bit != 1) {
+ ++_memory_page_size_shift;
+ page_size_bit >>= 1;
+ }
+ _memory_page_size = ((size_t)1 << _memory_page_size_shift);
+
+#if RPMALLOC_CONFIGURABLE
+ if (!_memory_config.span_size) {
+ _memory_span_size = _memory_default_span_size;
+ _memory_span_size_shift = _memory_default_span_size_shift;
+ _memory_span_mask = _memory_default_span_mask;
+ } else {
+ size_t span_size = _memory_config.span_size;
+ if (span_size > (256 * 1024))
+ span_size = (256 * 1024);
+ _memory_span_size = 4096;
+ _memory_span_size_shift = 12;
+ while (_memory_span_size < span_size) {
+ _memory_span_size <<= 1;
+ ++_memory_span_size_shift;
+ }
+ _memory_span_mask = ~(uintptr_t)(_memory_span_size - 1);
+ }
+#endif
+
+ _memory_span_map_count =
+ (_memory_config.span_map_count ? _memory_config.span_map_count
+ : DEFAULT_SPAN_MAP_COUNT);
+ if ((_memory_span_size * _memory_span_map_count) < _memory_page_size)
+ _memory_span_map_count = (_memory_page_size / _memory_span_size);
+ if ((_memory_page_size >= _memory_span_size) &&
+ ((_memory_span_map_count * _memory_span_size) % _memory_page_size))
+ _memory_span_map_count = (_memory_page_size / _memory_span_size);
+ _memory_heap_reserve_count = (_memory_span_map_count > DEFAULT_SPAN_MAP_COUNT)
+ ? DEFAULT_SPAN_MAP_COUNT
+ : _memory_span_map_count;
+
+ _memory_config.page_size = _memory_page_size;
+ _memory_config.span_size = _memory_span_size;
+ _memory_config.span_map_count = _memory_span_map_count;
+ _memory_config.enable_huge_pages = _memory_huge_pages;
+
+#if ((defined(__APPLE__) || defined(__HAIKU__)) && ENABLE_PRELOAD) || \
+ defined(__TINYC__)
+ if (pthread_key_create(&_memory_thread_heap, _rpmalloc_heap_release_raw_fc))
+ return -1;
+#endif
+#if defined(_WIN32) && (!defined(BUILD_DYNAMIC_LINK) || !BUILD_DYNAMIC_LINK)
+ fls_key = FlsAlloc(&_rpmalloc_thread_destructor);
+#endif
+
+ // Setup all small and medium size classes
+ size_t iclass = 0;
+ _memory_size_class[iclass].block_size = SMALL_GRANULARITY;
+ _rpmalloc_adjust_size_class(iclass);
+ for (iclass = 1; iclass < SMALL_CLASS_COUNT; ++iclass) {
+ size_t size = iclass * SMALL_GRANULARITY;
+ _memory_size_class[iclass].block_size = (uint32_t)size;
+ _rpmalloc_adjust_size_class(iclass);
+ }
+ // At least two blocks per span, then fall back to large allocations
+ _memory_medium_size_limit = (_memory_span_size - SPAN_HEADER_SIZE) >> 1;
+ if (_memory_medium_size_limit > MEDIUM_SIZE_LIMIT)
+ _memory_medium_size_limit = MEDIUM_SIZE_LIMIT;
+ for (iclass = 0; iclass < MEDIUM_CLASS_COUNT; ++iclass) {
+ size_t size = SMALL_SIZE_LIMIT + ((iclass + 1) * MEDIUM_GRANULARITY);
+ if (size > _memory_medium_size_limit)
+ break;
+ _memory_size_class[SMALL_CLASS_COUNT + iclass].block_size = (uint32_t)size;
+ _rpmalloc_adjust_size_class(SMALL_CLASS_COUNT + iclass);
+ }
+
+ _memory_orphan_heaps = 0;
+#if RPMALLOC_FIRST_CLASS_HEAPS
+ _memory_first_class_orphan_heaps = 0;
+#endif
+#if ENABLE_STATISTICS
+ atomic_store32(&_memory_active_heaps, 0);
+ atomic_store32(&_mapped_pages, 0);
+ _mapped_pages_peak = 0;
+ atomic_store32(&_master_spans, 0);
+ atomic_store32(&_mapped_total, 0);
+ atomic_store32(&_unmapped_total, 0);
+ atomic_store32(&_mapped_pages_os, 0);
+ atomic_store32(&_huge_pages_current, 0);
+ _huge_pages_peak = 0;
+#endif
+ memset(_memory_heaps, 0, sizeof(_memory_heaps));
+ atomic_store32_release(&_memory_global_lock, 0);
+
+ // Initialize this thread
+ rpmalloc_thread_initialize();
+ return 0;
+}
+
+//! Finalize the allocator
+void rpmalloc_finalize(void) {
+ rpmalloc_thread_finalize(1);
+ // rpmalloc_dump_statistics(stdout);
+
+ if (_memory_global_reserve) {
+ atomic_add32(&_memory_global_reserve_master->remaining_spans,
+ -(int32_t)_memory_global_reserve_count);
+ _memory_global_reserve_master = 0;
+ _memory_global_reserve_count = 0;
+ _memory_global_reserve = 0;
+ }
+ atomic_store32_release(&_memory_global_lock, 0);
+
+ // Free all thread caches and fully free spans
+ for (size_t list_idx = 0; list_idx < HEAP_ARRAY_SIZE; ++list_idx) {
+ heap_t *heap = _memory_heaps[list_idx];
+ while (heap) {
+ heap_t *next_heap = heap->next_heap;
+ heap->finalize = 1;
+ _rpmalloc_heap_global_finalize(heap);
+ heap = next_heap;
+ }
+ }
+
+#if ENABLE_GLOBAL_CACHE
+ // Free global caches
+ for (size_t iclass = 0; iclass < LARGE_CLASS_COUNT; ++iclass)
+ _rpmalloc_global_cache_finalize(&_memory_span_cache[iclass]);
+#endif
+
+#if (defined(__APPLE__) || defined(__HAIKU__)) && ENABLE_PRELOAD
+ pthread_key_delete(_memory_thread_heap);
+#endif
+#if defined(_WIN32) && (!defined(BUILD_DYNAMIC_LINK) || !BUILD_DYNAMIC_LINK)
+ FlsFree(fls_key);
+ fls_key = 0;
+#endif
+#if ENABLE_STATISTICS
+ // If you hit these asserts you probably have memory leaks (perhaps global
+ // scope data doing dynamic allocations) or double frees in your code
+ rpmalloc_assert(atomic_load32(&_mapped_pages) == 0, "Memory leak detected");
+ rpmalloc_assert(atomic_load32(&_mapped_pages_os) == 0,
+ "Memory leak detected");
+#endif
+
+ _rpmalloc_initialized = 0;
+}
+
+//! Initialize thread, assign heap
+extern inline void rpmalloc_thread_initialize(void) {
+ if (!get_thread_heap_raw()) {
+ heap_t *heap = _rpmalloc_heap_allocate(0);
+ if (heap) {
+ _rpmalloc_stat_inc(&_memory_active_heaps);
+ set_thread_heap(heap);
+#if defined(_WIN32) && (!defined(BUILD_DYNAMIC_LINK) || !BUILD_DYNAMIC_LINK)
+ FlsSetValue(fls_key, heap);
+#endif
+ }
+ }
+}
+
+//! Finalize thread, orphan heap
+void rpmalloc_thread_finalize(int release_caches) {
+ heap_t *heap = get_thread_heap_raw();
+ if (heap)
+ _rpmalloc_heap_release_raw(heap, release_caches);
+ set_thread_heap(0);
+#if defined(_WIN32) && (!defined(BUILD_DYNAMIC_LINK) || !BUILD_DYNAMIC_LINK)
+ FlsSetValue(fls_key, 0);
+#endif
+}
+
+int rpmalloc_is_thread_initialized(void) {
+ return (get_thread_heap_raw() != 0) ? 1 : 0;
+}
+
+const rpmalloc_config_t *rpmalloc_config(void) { return &_memory_config; }
+
+// Extern interface
+
+extern inline RPMALLOC_ALLOCATOR void *rpmalloc(size_t size) {
+#if ENABLE_VALIDATE_ARGS
+ if (size >= MAX_ALLOC_SIZE) {
+ errno = EINVAL;
+ return 0;
+ }
+#endif
+ heap_t *heap = get_thread_heap();
+ return _rpmalloc_allocate(heap, size);
+}
+
+extern inline void rpfree(void *ptr) { _rpmalloc_deallocate(ptr); }
+
+extern inline RPMALLOC_ALLOCATOR void *rpcalloc(size_t num, size_t size) {
+ size_t total;
+#if ENABLE_VALIDATE_ARGS
+#if PLATFORM_WINDOWS
+ int err = SizeTMult(num, size, &total);
+ if ((err != S_OK) || (total >= MAX_ALLOC_SIZE)) {
+ errno = EINVAL;
+ return 0;
+ }
+#else
+ int err = __builtin_umull_overflow(num, size, &total);
+ if (err || (total >= MAX_ALLOC_SIZE)) {
+ errno = EINVAL;
+ return 0;
+ }
+#endif
+#else
+ total = num * size;
+#endif
+ heap_t *heap = get_thread_heap();
+ void *block = _rpmalloc_allocate(heap, total);
+ if (block)
+ memset(block, 0, total);
+ return block;
+}
+
+extern inline RPMALLOC_ALLOCATOR void *rprealloc(void *ptr, size_t size) {
+#if ENABLE_VALIDATE_ARGS
+ if (size >= MAX_ALLOC_SIZE) {
+ errno = EINVAL;
+ return ptr;
+ }
+#endif
+ heap_t *heap = get_thread_heap();
+ return _rpmalloc_reallocate(heap, ptr, size, 0, 0);
+}
+
+extern RPMALLOC_ALLOCATOR void *rpaligned_realloc(void *ptr, size_t alignment,
+ size_t size, size_t oldsize,
+ unsigned int flags) {
+#if ENABLE_VALIDATE_ARGS
+ if ((size + alignment < size) || (alignment > _memory_page_size)) {
+ errno = EINVAL;
+ return 0;
+ }
+#endif
+ heap_t *heap = get_thread_heap();
+ return _rpmalloc_aligned_reallocate(heap, ptr, alignment, size, oldsize,
+ flags);
+}
+
+extern RPMALLOC_ALLOCATOR void *rpaligned_alloc(size_t alignment, size_t size) {
+ heap_t *heap = get_thread_heap();
+ return _rpmalloc_aligned_allocate(heap, alignment, size);
+}
+
+extern inline RPMALLOC_ALLOCATOR void *
+rpaligned_calloc(size_t alignment, size_t num, size_t size) {
+ size_t total;
+#if ENABLE_VALIDATE_ARGS
+#if PLATFORM_WINDOWS
+ int err = SizeTMult(num, size, &total);
+ if ((err != S_OK) || (total >= MAX_ALLOC_SIZE)) {
+ errno = EINVAL;
+ return 0;
+ }
+#else
+ int err = __builtin_umull_overflow(num, size, &total);
+ if (err || (total >= MAX_ALLOC_SIZE)) {
+ errno = EINVAL;
+ return 0;
+ }
+#endif
+#else
+ total = num * size;
+#endif
+ void *block = rpaligned_alloc(alignment, total);
+ if (block)
+ memset(block, 0, total);
+ return block;
+}
+
+extern inline RPMALLOC_ALLOCATOR void *rpmemalign(size_t alignment,
+ size_t size) {
+ return rpaligned_alloc(alignment, size);
+}
+
+extern inline int rpposix_memalign(void **memptr, size_t alignment,
+ size_t size) {
+ if (memptr)
+ *memptr = rpaligned_alloc(alignment, size);
+ else
+ return EINVAL;
+ return *memptr ? 0 : ENOMEM;
+}
+
+extern inline size_t rpmalloc_usable_size(void *ptr) {
+ return (ptr ? _rpmalloc_usable_size(ptr) : 0);
+}
+
+extern inline void rpmalloc_thread_collect(void) {}
+
+void rpmalloc_thread_statistics(rpmalloc_thread_statistics_t *stats) {
+ memset(stats, 0, sizeof(rpmalloc_thread_statistics_t));
+ heap_t *heap = get_thread_heap_raw();
+ if (!heap)
+ return;
+
+ for (size_t iclass = 0; iclass < SIZE_CLASS_COUNT; ++iclass) {
+ size_class_t *size_class = _memory_size_class + iclass;
+ span_t *span = heap->size_class[iclass].partial_span;
+ while (span) {
+ size_t free_count = span->list_size;
+ size_t block_count = size_class->block_count;
+ if (span->free_list_limit < block_count)
+ block_count = span->free_list_limit;
+ free_count += (block_count - span->used_count);
+ stats->sizecache = free_count * size_class->block_size;
+ span = span->next;
+ }
+ }
+
+#if ENABLE_THREAD_CACHE
+ for (size_t iclass = 0; iclass < LARGE_CLASS_COUNT; ++iclass) {
+ span_cache_t *span_cache;
+ if (!iclass)
+ span_cache = &heap->span_cache;
+ else
+ span_cache = (span_cache_t *)(heap->span_large_cache + (iclass - 1));
+ stats->spancache = span_cache->count * (iclass + 1) * _memory_span_size;
+ }
+#endif
+
+ span_t *deferred = (span_t *)atomic_load_ptr(&heap->span_free_deferred);
+ while (deferred) {
+ if (deferred->size_class != SIZE_CLASS_HUGE)
+ stats->spancache = (size_t)deferred->span_count * _memory_span_size;
+ deferred = (span_t *)deferred->free_list;
+ }
+
+#if ENABLE_STATISTICS
+ stats->thread_to_global = (size_t)atomic_load64(&heap->thread_to_global);
+ stats->global_to_thread = (size_t)atomic_load64(&heap->global_to_thread);
+
+ for (size_t iclass = 0; iclass < LARGE_CLASS_COUNT; ++iclass) {
+ stats->span_use[iclass].current =
+ (size_t)atomic_load32(&heap->span_use[iclass].current);
+ stats->span_use[iclass].peak =
+ (size_t)atomic_load32(&heap->span_use[iclass].high);
+ stats->span_use[iclass].to_global =
+ (size_t)atomic_load32(&heap->span_use[iclass].spans_to_global);
+ stats->span_use[iclass].from_global =
+ (size_t)atomic_load32(&heap->span_use[iclass].spans_from_global);
+ stats->span_use[iclass].to_cache =
+ (size_t)atomic_load32(&heap->span_use[iclass].spans_to_cache);
+ stats->span_use[iclass].from_cache =
+ (size_t)atomic_load32(&heap->span_use[iclass].spans_from_cache);
+ stats->span_use[iclass].to_reserved =
+ (size_t)atomic_load32(&heap->span_use[iclass].spans_to_reserved);
+ stats->span_use[iclass].from_reserved =
+ (size_t)atomic_load32(&heap->span_use[iclass].spans_from_reserved);
+ stats->span_use[iclass].map_calls =
+ (size_t)atomic_load32(&heap->span_use[iclass].spans_map_calls);
+ }
+ for (size_t iclass = 0; iclass < SIZE_CLASS_COUNT; ++iclass) {
+ stats->size_use[iclass].alloc_current =
+ (size_t)atomic_load32(&heap->size_class_use[iclass].alloc_current);
+ stats->size_use[iclass].alloc_peak =
+ (size_t)heap->size_class_use[iclass].alloc_peak;
+ stats->size_use[iclass].alloc_total =
+ (size_t)atomic_load32(&heap->size_class_use[iclass].alloc_total);
+ stats->size_use[iclass].free_total =
+ (size_t)atomic_load32(&heap->size_class_use[iclass].free_total);
+ stats->size_use[iclass].spans_to_cache =
+ (size_t)atomic_load32(&heap->size_class_use[iclass].spans_to_cache);
+ stats->size_use[iclass].spans_from_cache =
+ (size_t)atomic_load32(&heap->size_class_use[iclass].spans_from_cache);
+ stats->size_use[iclass].spans_from_reserved = (size_t)atomic_load32(
+ &heap->size_class_use[iclass].spans_from_reserved);
+ stats->size_use[iclass].map_calls =
+ (size_t)atomic_load32(&heap->size_class_use[iclass].spans_map_calls);
+ }
+#endif
+}
+
+void rpmalloc_global_statistics(rpmalloc_global_statistics_t *stats) {
+ memset(stats, 0, sizeof(rpmalloc_global_statistics_t));
+#if ENABLE_STATISTICS
+ stats->mapped = (size_t)atomic_load32(&_mapped_pages) * _memory_page_size;
+ stats->mapped_peak = (size_t)_mapped_pages_peak * _memory_page_size;
+ stats->mapped_total =
+ (size_t)atomic_load32(&_mapped_total) * _memory_page_size;
+ stats->unmapped_total =
+ (size_t)atomic_load32(&_unmapped_total) * _memory_page_size;
+ stats->huge_alloc =
+ (size_t)atomic_load32(&_huge_pages_current) * _memory_page_size;
+ stats->huge_alloc_peak = (size_t)_huge_pages_peak * _memory_page_size;
+#endif
+#if ENABLE_GLOBAL_CACHE
+ for (size_t iclass = 0; iclass < LARGE_CLASS_COUNT; ++iclass)
+ stats->cached +=
+ _memory_span_cache[iclass].count * (iclass + 1) * _memory_span_size;
+#endif
+}
+
+#if ENABLE_STATISTICS
+
+static void _memory_heap_dump_statistics(heap_t *heap, void *file) {
+ fprintf(file, "Heap %d stats:\n", heap->id);
+ fprintf(file, "Class CurAlloc PeakAlloc TotAlloc TotFree BlkSize "
+ "BlkCount SpansCur SpansPeak PeakAllocMiB ToCacheMiB "
+ "FromCacheMiB FromReserveMiB MmapCalls\n");
+ for (size_t iclass = 0; iclass < SIZE_CLASS_COUNT; ++iclass) {
+ if (!atomic_load32(&heap->size_class_use[iclass].alloc_total))
+ continue;
+ fprintf(
+ file,
+ "%3u: %10u %10u %10u %10u %8u %8u %8d %9d %13zu %11zu %12zu %14zu "
+ "%9u\n",
+ (uint32_t)iclass,
+ atomic_load32(&heap->size_class_use[iclass].alloc_current),
+ heap->size_class_use[iclass].alloc_peak,
+ atomic_load32(&heap->size_class_use[iclass].alloc_total),
+ atomic_load32(&heap->size_class_use[iclass].free_total),
+ _memory_size_class[iclass].block_size,
+ _memory_size_class[iclass].block_count,
+ atomic_load32(&heap->size_class_use[iclass].spans_current),
+ heap->size_class_use[iclass].spans_peak,
+ ((size_t)heap->size_class_use[iclass].alloc_peak *
+ (size_t)_memory_size_class[iclass].block_size) /
+ (size_t)(1024 * 1024),
+ ((size_t)atomic_load32(&heap->size_class_use[iclass].spans_to_cache) *
+ _memory_span_size) /
+ (size_t)(1024 * 1024),
+ ((size_t)atomic_load32(&heap->size_class_use[iclass].spans_from_cache) *
+ _memory_span_size) /
+ (size_t)(1024 * 1024),
+ ((size_t)atomic_load32(
+ &heap->size_class_use[iclass].spans_from_reserved) *
+ _memory_span_size) /
+ (size_t)(1024 * 1024),
+ atomic_load32(&heap->size_class_use[iclass].spans_map_calls));
+ }
+ fprintf(file, "Spans Current Peak Deferred PeakMiB Cached ToCacheMiB "
+ "FromCacheMiB ToReserveMiB FromReserveMiB ToGlobalMiB "
+ "FromGlobalMiB MmapCalls\n");
+ for (size_t iclass = 0; iclass < LARGE_CLASS_COUNT; ++iclass) {
+ if (!atomic_load32(&heap->span_use[iclass].high) &&
+ !atomic_load32(&heap->span_use[iclass].spans_map_calls))
+ continue;
+ fprintf(
+ file,
+ "%4u: %8d %8u %8u %8zu %7u %11zu %12zu %12zu %14zu %11zu %13zu %10u\n",
+ (uint32_t)(iclass + 1), atomic_load32(&heap->span_use[iclass].current),
+ atomic_load32(&heap->span_use[iclass].high),
+ atomic_load32(&heap->span_use[iclass].spans_deferred),
+ ((size_t)atomic_load32(&heap->span_use[iclass].high) *
+ (size_t)_memory_span_size * (iclass + 1)) /
+ (size_t)(1024 * 1024),
+#if ENABLE_THREAD_CACHE
+ (unsigned int)(!iclass ? heap->span_cache.count
+ : heap->span_large_cache[iclass - 1].count),
+ ((size_t)atomic_load32(&heap->span_use[iclass].spans_to_cache) *
+ (iclass + 1) * _memory_span_size) /
+ (size_t)(1024 * 1024),
+ ((size_t)atomic_load32(&heap->span_use[iclass].spans_from_cache) *
+ (iclass + 1) * _memory_span_size) /
+ (size_t)(1024 * 1024),
+#else
+ 0, (size_t)0, (size_t)0,
+#endif
+ ((size_t)atomic_load32(&heap->span_use[iclass].spans_to_reserved) *
+ (iclass + 1) * _memory_span_size) /
+ (size_t)(1024 * 1024),
+ ((size_t)atomic_load32(&heap->span_use[iclass].spans_from_reserved) *
+ (iclass + 1) * _memory_span_size) /
+ (size_t)(1024 * 1024),
+ ((size_t)atomic_load32(&heap->span_use[iclass].spans_to_global) *
+ (size_t)_memory_span_size * (iclass + 1)) /
+ (size_t)(1024 * 1024),
+ ((size_t)atomic_load32(&heap->span_use[iclass].spans_from_global) *
+ (size_t)_memory_span_size * (iclass + 1)) /
+ (size_t)(1024 * 1024),
+ atomic_load32(&heap->span_use[iclass].spans_map_calls));
+ }
+ fprintf(file, "Full spans: %zu\n", heap->full_span_count);
+ fprintf(file, "ThreadToGlobalMiB GlobalToThreadMiB\n");
+ fprintf(
+ file, "%17zu %17zu\n",
+ (size_t)atomic_load64(&heap->thread_to_global) / (size_t)(1024 * 1024),
+ (size_t)atomic_load64(&heap->global_to_thread) / (size_t)(1024 * 1024));
+}
+
+#endif
+
+void rpmalloc_dump_statistics(void *file) {
+#if ENABLE_STATISTICS
+ for (size_t list_idx = 0; list_idx < HEAP_ARRAY_SIZE; ++list_idx) {
+ heap_t *heap = _memory_heaps[list_idx];
+ while (heap) {
+ int need_dump = 0;
+ for (size_t iclass = 0; !need_dump && (iclass < SIZE_CLASS_COUNT);
+ ++iclass) {
+ if (!atomic_load32(&heap->size_class_use[iclass].alloc_total)) {
+ rpmalloc_assert(
+ !atomic_load32(&heap->size_class_use[iclass].free_total),
+ "Heap statistics counter mismatch");
+ rpmalloc_assert(
+ !atomic_load32(&heap->size_class_use[iclass].spans_map_calls),
+ "Heap statistics counter mismatch");
+ continue;
+ }
+ need_dump = 1;
+ }
+ for (size_t iclass = 0; !need_dump && (iclass < LARGE_CLASS_COUNT);
+ ++iclass) {
+ if (!atomic_load32(&heap->span_use[iclass].high) &&
+ !atomic_load32(&heap->span_use[iclass].spans_map_calls))
+ continue;
+ need_dump = 1;
+ }
+ if (need_dump)
+ _memory_heap_dump_statistics(heap, file);
+ heap = heap->next_heap;
+ }
+ }
+ fprintf(file, "Global stats:\n");
+ size_t huge_current =
+ (size_t)atomic_load32(&_huge_pages_current) * _memory_page_size;
+ size_t huge_peak = (size_t)_huge_pages_peak * _memory_page_size;
+ fprintf(file, "HugeCurrentMiB HugePeakMiB\n");
+ fprintf(file, "%14zu %11zu\n", huge_current / (size_t)(1024 * 1024),
+ huge_peak / (size_t)(1024 * 1024));
+
+ fprintf(file, "GlobalCacheMiB\n");
+ for (size_t iclass = 0; iclass < LARGE_CLASS_COUNT; ++iclass) {
+ global_cache_t *cache = _memory_span_cache + iclass;
+ size_t global_cache = (size_t)cache->count * iclass * _memory_span_size;
+
+ size_t global_overflow_cache = 0;
+ span_t *span = cache->overflow;
+ while (span) {
+ global_overflow_cache += iclass * _memory_span_size;
+ span = span->next;
+ }
+ if (global_cache || global_overflow_cache || cache->insert_count ||
+ cache->extract_count)
+ fprintf(file,
+ "%4zu: %8zuMiB (%8zuMiB overflow) %14zu insert %14zu extract\n",
+ iclass + 1, global_cache / (size_t)(1024 * 1024),
+ global_overflow_cache / (size_t)(1024 * 1024),
+ cache->insert_count, cache->extract_count);
+ }
+
+ size_t mapped = (size_t)atomic_load32(&_mapped_pages) * _memory_page_size;
+ size_t mapped_os =
+ (size_t)atomic_load32(&_mapped_pages_os) * _memory_page_size;
+ size_t mapped_peak = (size_t)_mapped_pages_peak * _memory_page_size;
+ size_t mapped_total =
+ (size_t)atomic_load32(&_mapped_total) * _memory_page_size;
+ size_t unmapped_total =
+ (size_t)atomic_load32(&_unmapped_total) * _memory_page_size;
+ fprintf(
+ file,
+ "MappedMiB MappedOSMiB MappedPeakMiB MappedTotalMiB UnmappedTotalMiB\n");
+ fprintf(file, "%9zu %11zu %13zu %14zu %16zu\n",
+ mapped / (size_t)(1024 * 1024), mapped_os / (size_t)(1024 * 1024),
+ mapped_peak / (size_t)(1024 * 1024),
+ mapped_total / (size_t)(1024 * 1024),
+ unmapped_total / (size_t)(1024 * 1024));
+
+ fprintf(file, "\n");
+#if 0
+ int64_t allocated = atomic_load64(&_allocation_counter);
+ int64_t deallocated = atomic_load64(&_deallocation_counter);
+ fprintf(file, "Allocation count: %lli\n", allocated);
+ fprintf(file, "Deallocation count: %lli\n", deallocated);
+ fprintf(file, "Current allocations: %lli\n", (allocated - deallocated));
+ fprintf(file, "Master spans: %d\n", atomic_load32(&_master_spans));
+ fprintf(file, "Dangling master spans: %d\n", atomic_load32(&_unmapped_master_spans));
+#endif
+#endif
+ (void)sizeof(file);
+}
+
+#if RPMALLOC_FIRST_CLASS_HEAPS
+
+extern inline rpmalloc_heap_t *rpmalloc_heap_acquire(void) {
+ // Must be a pristine heap from newly mapped memory pages, or else memory
+ // blocks could already be allocated from the heap which would (wrongly) be
+ // released when heap is cleared with rpmalloc_heap_free_all(). Also heaps
+ // guaranteed to be pristine from the dedicated orphan list can be used.
+ heap_t *heap = _rpmalloc_heap_allocate(1);
+ heap->owner_thread = 0;
+ _rpmalloc_stat_inc(&_memory_active_heaps);
+ return heap;
+}
+
+extern inline void rpmalloc_heap_release(rpmalloc_heap_t *heap) {
+ if (heap)
+ _rpmalloc_heap_release(heap, 1, 1);
+}
+
+extern inline RPMALLOC_ALLOCATOR void *
+rpmalloc_heap_alloc(rpmalloc_heap_t *heap, size_t size) {
+#if ENABLE_VALIDATE_ARGS
+ if (size >= MAX_ALLOC_SIZE) {
+ errno = EINVAL;
+ return 0;
+ }
+#endif
+ return _rpmalloc_allocate(heap, size);
+}
+
+extern inline RPMALLOC_ALLOCATOR void *
+rpmalloc_heap_aligned_alloc(rpmalloc_heap_t *heap, size_t alignment,
+ size_t size) {
+#if ENABLE_VALIDATE_ARGS
+ if (size >= MAX_ALLOC_SIZE) {
+ errno = EINVAL;
+ return 0;
+ }
+#endif
+ return _rpmalloc_aligned_allocate(heap, alignment, size);
+}
+
+extern inline RPMALLOC_ALLOCATOR void *
+rpmalloc_heap_calloc(rpmalloc_heap_t *heap, size_t num, size_t size) {
+ return rpmalloc_heap_aligned_calloc(heap, 0, num, size);
+}
+
+extern inline RPMALLOC_ALLOCATOR void *
+rpmalloc_heap_aligned_calloc(rpmalloc_heap_t *heap, size_t alignment,
+ size_t num, size_t size) {
+ size_t total;
+#if ENABLE_VALIDATE_ARGS
+#if PLATFORM_WINDOWS
+ int err = SizeTMult(num, size, &total);
+ if ((err != S_OK) || (total >= MAX_ALLOC_SIZE)) {
+ errno = EINVAL;
+ return 0;
+ }
+#else
+ int err = __builtin_umull_overflow(num, size, &total);
+ if (err || (total >= MAX_ALLOC_SIZE)) {
+ errno = EINVAL;
+ return 0;
+ }
+#endif
+#else
+ total = num * size;
+#endif
+ void *block = _rpmalloc_aligned_allocate(heap, alignment, total);
+ if (block)
+ memset(block, 0, total);
+ return block;
+}
+
+extern inline RPMALLOC_ALLOCATOR void *
+rpmalloc_heap_realloc(rpmalloc_heap_t *heap, void *ptr, size_t size,
+ unsigned int flags) {
+#if ENABLE_VALIDATE_ARGS
+ if (size >= MAX_ALLOC_SIZE) {
+ errno = EINVAL;
+ return ptr;
+ }
+#endif
+ return _rpmalloc_reallocate(heap, ptr, size, 0, flags);
+}
+
+extern inline RPMALLOC_ALLOCATOR void *
+rpmalloc_heap_aligned_realloc(rpmalloc_heap_t *heap, void *ptr,
+ size_t alignment, size_t size,
+ unsigned int flags) {
+#if ENABLE_VALIDATE_ARGS
+ if ((size + alignment < size) || (alignment > _memory_page_size)) {
+ errno = EINVAL;
+ return 0;
+ }
+#endif
+ return _rpmalloc_aligned_reallocate(heap, ptr, alignment, size, 0, flags);
+}
+
+extern inline void rpmalloc_heap_free(rpmalloc_heap_t *heap, void *ptr) {
+ (void)sizeof(heap);
+ _rpmalloc_deallocate(ptr);
+}
+
+extern inline void rpmalloc_heap_free_all(rpmalloc_heap_t *heap) {
+ span_t *span;
+ span_t *next_span;
+
+ _rpmalloc_heap_cache_adopt_deferred(heap, 0);
+
+ for (size_t iclass = 0; iclass < SIZE_CLASS_COUNT; ++iclass) {
+ span = heap->size_class[iclass].partial_span;
+ while (span) {
+ next_span = span->next;
+ _rpmalloc_heap_cache_insert(heap, span);
+ span = next_span;
+ }
+ heap->size_class[iclass].partial_span = 0;
+ span = heap->full_span[iclass];
+ while (span) {
+ next_span = span->next;
+ _rpmalloc_heap_cache_insert(heap, span);
+ span = next_span;
+ }
+ }
+ memset(heap->size_class, 0, sizeof(heap->size_class));
+ memset(heap->full_span, 0, sizeof(heap->full_span));
+
+ span = heap->large_huge_span;
+ while (span) {
+ next_span = span->next;
+ if (UNEXPECTED(span->size_class == SIZE_CLASS_HUGE))
+ _rpmalloc_deallocate_huge(span);
+ else
+ _rpmalloc_heap_cache_insert(heap, span);
+ span = next_span;
+ }
+ heap->large_huge_span = 0;
+ heap->full_span_count = 0;
+
+#if ENABLE_THREAD_CACHE
+ for (size_t iclass = 0; iclass < LARGE_CLASS_COUNT; ++iclass) {
+ span_cache_t *span_cache;
+ if (!iclass)
+ span_cache = &heap->span_cache;
+ else
+ span_cache = (span_cache_t *)(heap->span_large_cache + (iclass - 1));
+ if (!span_cache->count)
+ continue;
+#if ENABLE_GLOBAL_CACHE
+ _rpmalloc_stat_add64(&heap->thread_to_global,
+ span_cache->count * (iclass + 1) * _memory_span_size);
+ _rpmalloc_stat_add(&heap->span_use[iclass].spans_to_global,
+ span_cache->count);
+ _rpmalloc_global_cache_insert_spans(span_cache->span, iclass + 1,
+ span_cache->count);
+#else
+ for (size_t ispan = 0; ispan < span_cache->count; ++ispan)
+ _rpmalloc_span_unmap(span_cache->span[ispan]);
+#endif
+ span_cache->count = 0;
+ }
+#endif
+
+#if ENABLE_STATISTICS
+ for (size_t iclass = 0; iclass < SIZE_CLASS_COUNT; ++iclass) {
+ atomic_store32(&heap->size_class_use[iclass].alloc_current, 0);
+ atomic_store32(&heap->size_class_use[iclass].spans_current, 0);
+ }
+ for (size_t iclass = 0; iclass < LARGE_CLASS_COUNT; ++iclass) {
+ atomic_store32(&heap->span_use[iclass].current, 0);
+ }
+#endif
+}
+
+extern inline void rpmalloc_heap_thread_set_current(rpmalloc_heap_t *heap) {
+ heap_t *prev_heap = get_thread_heap_raw();
+ if (prev_heap != heap) {
+ set_thread_heap(heap);
+ if (prev_heap)
+ rpmalloc_heap_release(prev_heap);
+ }
+}
+
+#endif
+
+#if ENABLE_PRELOAD || ENABLE_OVERRIDE
+
+#include "malloc.c"
+
+#endif
diff --git a/llvm/lib/Support/rpmalloc/rpmalloc.h b/llvm/lib/Support/rpmalloc/rpmalloc.h
new file mode 100644
index 0000000000000..a57bc4a6a2ece
--- /dev/null
+++ b/llvm/lib/Support/rpmalloc/rpmalloc.h
@@ -0,0 +1,422 @@
+//===----------------------- rpmalloc.h -------------------*- C -*-===========//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This library provides a cross-platform lock free thread caching malloc
+// implementation in C11.
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if defined(__clang__) || defined(__GNUC__)
+#define RPMALLOC_EXPORT __attribute__((visibility("default")))
+#define RPMALLOC_ALLOCATOR
+#if (defined(__clang_major__) && (__clang_major__ < 4)) || \
+ (defined(__GNUC__) && defined(ENABLE_PRELOAD) && ENABLE_PRELOAD)
+#define RPMALLOC_ATTRIB_MALLOC
+#define RPMALLOC_ATTRIB_ALLOC_SIZE(size)
+#define RPMALLOC_ATTRIB_ALLOC_SIZE2(count, size)
+#else
+#define RPMALLOC_ATTRIB_MALLOC __attribute__((__malloc__))
+#define RPMALLOC_ATTRIB_ALLOC_SIZE(size) __attribute__((alloc_size(size)))
+#define RPMALLOC_ATTRIB_ALLOC_SIZE2(count, size) \
+ __attribute__((alloc_size(count, size)))
+#endif
+#define RPMALLOC_CDECL
+#elif defined(_MSC_VER)
+#define RPMALLOC_EXPORT
+#define RPMALLOC_ALLOCATOR __declspec(allocator) __declspec(restrict)
+#define RPMALLOC_ATTRIB_MALLOC
+#define RPMALLOC_ATTRIB_ALLOC_SIZE(size)
+#define RPMALLOC_ATTRIB_ALLOC_SIZE2(count, size)
+#define RPMALLOC_CDECL __cdecl
+#else
+#define RPMALLOC_EXPORT
+#define RPMALLOC_ALLOCATOR
+#define RPMALLOC_ATTRIB_MALLOC
+#define RPMALLOC_ATTRIB_ALLOC_SIZE(size)
+#define RPMALLOC_ATTRIB_ALLOC_SIZE2(count, size)
+#define RPMALLOC_CDECL
+#endif
+
+//! Define RPMALLOC_CONFIGURABLE to enable configuring sizes. Will introduce
+// a very small overhead due to some size calculations not being compile time
+// constants
+#ifndef RPMALLOC_CONFIGURABLE
+#define RPMALLOC_CONFIGURABLE 0
+#endif
+
+//! Define RPMALLOC_FIRST_CLASS_HEAPS to enable heap based API (rpmalloc_heap_*
+//! functions).
+// Will introduce a very small overhead to track fully allocated spans in heaps
+#ifndef RPMALLOC_FIRST_CLASS_HEAPS
+#define RPMALLOC_FIRST_CLASS_HEAPS 0
+#endif
+
+//! Flag to rpaligned_realloc to not preserve content in reallocation
+#define RPMALLOC_NO_PRESERVE 1
+//! Flag to rpaligned_realloc to fail and return null pointer if grow cannot be
+//! done in-place,
+// in which case the original pointer is still valid (just like a call to
+// realloc which failes to allocate a new block).
+#define RPMALLOC_GROW_OR_FAIL 2
+
+typedef struct rpmalloc_global_statistics_t {
+ //! Current amount of virtual memory mapped, all of which might not have been
+ //! committed (only if ENABLE_STATISTICS=1)
+ size_t mapped;
+ //! Peak amount of virtual memory mapped, all of which might not have been
+ //! committed (only if ENABLE_STATISTICS=1)
+ size_t mapped_peak;
+ //! Current amount of memory in global caches for small and medium sizes
+ //! (<32KiB)
+ size_t cached;
+ //! Current amount of memory allocated in huge allocations, i.e larger than
+ //! LARGE_SIZE_LIMIT which is 2MiB by default (only if ENABLE_STATISTICS=1)
+ size_t huge_alloc;
+ //! Peak amount of memory allocated in huge allocations, i.e larger than
+ //! LARGE_SIZE_LIMIT which is 2MiB by default (only if ENABLE_STATISTICS=1)
+ size_t huge_alloc_peak;
+ //! Total amount of memory mapped since initialization (only if
+ //! ENABLE_STATISTICS=1)
+ size_t mapped_total;
+ //! Total amount of memory unmapped since initialization (only if
+ //! ENABLE_STATISTICS=1)
+ size_t unmapped_total;
+} rpmalloc_global_statistics_t;
+
+typedef struct rpmalloc_thread_statistics_t {
+ //! Current number of bytes available in thread size class caches for small
+ //! and medium sizes (<32KiB)
+ size_t sizecache;
+ //! Current number of bytes available in thread span caches for small and
+ //! medium sizes (<32KiB)
+ size_t spancache;
+ //! Total number of bytes transitioned from thread cache to global cache (only
+ //! if ENABLE_STATISTICS=1)
+ size_t thread_to_global;
+ //! Total number of bytes transitioned from global cache to thread cache (only
+ //! if ENABLE_STATISTICS=1)
+ size_t global_to_thread;
+ //! Per span count statistics (only if ENABLE_STATISTICS=1)
+ struct {
+ //! Currently used number of spans
+ size_t current;
+ //! High water mark of spans used
+ size_t peak;
+ //! Number of spans transitioned to global cache
+ size_t to_global;
+ //! Number of spans transitioned from global cache
+ size_t from_global;
+ //! Number of spans transitioned to thread cache
+ size_t to_cache;
+ //! Number of spans transitioned from thread cache
+ size_t from_cache;
+ //! Number of spans transitioned to reserved state
+ size_t to_reserved;
+ //! Number of spans transitioned from reserved state
+ size_t from_reserved;
+ //! Number of raw memory map calls (not hitting the reserve spans but
+ //! resulting in actual OS mmap calls)
+ size_t map_calls;
+ } span_use[64];
+ //! Per size class statistics (only if ENABLE_STATISTICS=1)
+ struct {
+ //! Current number of allocations
+ size_t alloc_current;
+ //! Peak number of allocations
+ size_t alloc_peak;
+ //! Total number of allocations
+ size_t alloc_total;
+ //! Total number of frees
+ size_t free_total;
+ //! Number of spans transitioned to cache
+ size_t spans_to_cache;
+ //! Number of spans transitioned from cache
+ size_t spans_from_cache;
+ //! Number of spans transitioned from reserved state
+ size_t spans_from_reserved;
+ //! Number of raw memory map calls (not hitting the reserve spans but
+ //! resulting in actual OS mmap calls)
+ size_t map_calls;
+ } size_use[128];
+} rpmalloc_thread_statistics_t;
+
+typedef struct rpmalloc_config_t {
+ //! Map memory pages for the given number of bytes. The returned address MUST
+ //! be
+ // aligned to the rpmalloc span size, which will always be a power of two.
+ // Optionally the function can store an alignment offset in the offset
+ // variable in case it performs alignment and the returned pointer is offset
+ // from the actual start of the memory region due to this alignment. The
+ // alignment offset will be passed to the memory unmap function. The
+ // alignment offset MUST NOT be larger than 65535 (storable in an uint16_t),
+ // if it is you must use natural alignment to shift it into 16 bits. If you
+ // set a memory_map function, you must also set a memory_unmap function or
+ // else the default implementation will be used for both. This function must
+ // be thread safe, it can be called by multiple threads simultaneously.
+ void *(*memory_map)(size_t size, size_t *offset);
+ //! Unmap the memory pages starting at address and spanning the given number
+ //! of bytes.
+ // If release is set to non-zero, the unmap is for an entire span range as
+ // returned by a previous call to memory_map and that the entire range should
+ // be released. The release argument holds the size of the entire span range.
+ // If release is set to 0, the unmap is a partial decommit of a subset of the
+ // mapped memory range. If you set a memory_unmap function, you must also set
+ // a memory_map function or else the default implementation will be used for
+ // both. This function must be thread safe, it can be called by multiple
+ // threads simultaneously.
+ void (*memory_unmap)(void *address, size_t size, size_t offset,
+ size_t release);
+ //! Called when an assert fails, if asserts are enabled. Will use the standard
+ //! assert()
+ // if this is not set.
+ void (*error_callback)(const char *message);
+ //! Called when a call to map memory pages fails (out of memory). If this
+ //! callback is
+ // not set or returns zero the library will return a null pointer in the
+ // allocation call. If this callback returns non-zero the map call will be
+ // retried. The argument passed is the number of bytes that was requested in
+ // the map call. Only used if the default system memory map function is used
+ // (memory_map callback is not set).
+ int (*map_fail_callback)(size_t size);
+ //! Size of memory pages. The page size MUST be a power of two. All memory
+ //! mapping
+ // requests to memory_map will be made with size set to a multiple of the
+ // page size. Used if RPMALLOC_CONFIGURABLE is defined to 1, otherwise system
+ // page size is used.
+ size_t page_size;
+ //! Size of a span of memory blocks. MUST be a power of two, and in
+ //! [4096,262144]
+ // range (unless 0 - set to 0 to use the default span size). Used if
+ // RPMALLOC_CONFIGURABLE is defined to 1.
+ size_t span_size;
+ //! Number of spans to map at each request to map new virtual memory blocks.
+ //! This can
+ // be used to minimize the system call overhead at the cost of virtual memory
+ // address space. The extra mapped pages will not be written until actually
+ // used, so physical committed memory should not be affected in the default
+ // implementation. Will be aligned to a multiple of spans that match memory
+ // page size in case of huge pages.
+ size_t span_map_count;
+ //! Enable use of large/huge pages. If this flag is set to non-zero and page
+ //! size is
+ // zero, the allocator will try to enable huge pages and auto detect the
+ // configuration. If this is set to non-zero and page_size is also non-zero,
+ // the allocator will assume huge pages have been configured and enabled
+ // prior to initializing the allocator. For Windows, see
+ // https://docs.microsoft.com/en-us/windows/desktop/memory/large-page-support
+ // For Linux, see https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt
+ int enable_huge_pages;
+ //! Respectively allocated pages and huge allocated pages names for systems
+ // supporting it to be able to distinguish among anonymous regions.
+ const char *page_name;
+ const char *huge_page_name;
+} rpmalloc_config_t;
+
+//! Initialize allocator with default configuration
+RPMALLOC_EXPORT int rpmalloc_initialize(void);
+
+//! Initialize allocator with given configuration
+RPMALLOC_EXPORT int rpmalloc_initialize_config(const rpmalloc_config_t *config);
+
+//! Get allocator configuration
+RPMALLOC_EXPORT const rpmalloc_config_t *rpmalloc_config(void);
+
+//! Finalize allocator
+RPMALLOC_EXPORT void rpmalloc_finalize(void);
+
+//! Initialize allocator for calling thread
+RPMALLOC_EXPORT void rpmalloc_thread_initialize(void);
+
+//! Finalize allocator for calling thread
+RPMALLOC_EXPORT void rpmalloc_thread_finalize(int release_caches);
+
+//! Perform deferred deallocations pending for the calling thread heap
+RPMALLOC_EXPORT void rpmalloc_thread_collect(void);
+
+//! Query if allocator is initialized for calling thread
+RPMALLOC_EXPORT int rpmalloc_is_thread_initialized(void);
+
+//! Get per-thread statistics
+RPMALLOC_EXPORT void
+rpmalloc_thread_statistics(rpmalloc_thread_statistics_t *stats);
+
+//! Get global statistics
+RPMALLOC_EXPORT void
+rpmalloc_global_statistics(rpmalloc_global_statistics_t *stats);
+
+//! Dump all statistics in human readable format to file (should be a FILE*)
+RPMALLOC_EXPORT void rpmalloc_dump_statistics(void *file);
+
+//! Allocate a memory block of at least the given size
+RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *
+rpmalloc(size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(1);
+
+//! Free the given memory block
+RPMALLOC_EXPORT void rpfree(void *ptr);
+
+//! Allocate a memory block of at least the given size and zero initialize it
+RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *
+rpcalloc(size_t num, size_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE2(1, 2);
+
+//! Reallocate the given block to at least the given size
+RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *
+rprealloc(void *ptr, size_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(2);
+
+//! Reallocate the given block to at least the given size and alignment,
+// with optional control flags (see RPMALLOC_NO_PRESERVE).
+// Alignment must be a power of two and a multiple of sizeof(void*),
+// and should ideally be less than memory page size. A caveat of rpmalloc
+// internals is that this must also be strictly less than the span size
+// (default 64KiB)
+RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *
+rpaligned_realloc(void *ptr, size_t alignment, size_t size, size_t oldsize,
+ unsigned int flags) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(3);
+
+//! Allocate a memory block of at least the given size and alignment.
+// Alignment must be a power of two and a multiple of sizeof(void*),
+// and should ideally be less than memory page size. A caveat of rpmalloc
+// internals is that this must also be strictly less than the span size
+// (default 64KiB)
+RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *
+rpaligned_alloc(size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(2);
+
+//! Allocate a memory block of at least the given size and alignment, and zero
+//! initialize it.
+// Alignment must be a power of two and a multiple of sizeof(void*),
+// and should ideally be less than memory page size. A caveat of rpmalloc
+// internals is that this must also be strictly less than the span size
+// (default 64KiB)
+RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *
+rpaligned_calloc(size_t alignment, size_t num,
+ size_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE2(2, 3);
+
+//! Allocate a memory block of at least the given size and alignment.
+// Alignment must be a power of two and a multiple of sizeof(void*),
+// and should ideally be less than memory page size. A caveat of rpmalloc
+// internals is that this must also be strictly less than the span size
+// (default 64KiB)
+RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *
+rpmemalign(size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(2);
+
+//! Allocate a memory block of at least the given size and alignment.
+// Alignment must be a power of two and a multiple of sizeof(void*),
+// and should ideally be less than memory page size. A caveat of rpmalloc
+// internals is that this must also be strictly less than the span size
+// (default 64KiB)
+RPMALLOC_EXPORT int rpposix_memalign(void **memptr, size_t alignment,
+ size_t size);
+
+//! Query the usable size of the given memory block (from given pointer to the
+//! end of block)
+RPMALLOC_EXPORT size_t rpmalloc_usable_size(void *ptr);
+
+#if RPMALLOC_FIRST_CLASS_HEAPS
+
+//! Heap type
+typedef struct heap_t rpmalloc_heap_t;
+
+//! Acquire a new heap. Will reuse existing released heaps or allocate memory
+//! for a new heap
+// if none available. Heap API is implemented with the strict assumption that
+// only one single thread will call heap functions for a given heap at any
+// given time, no functions are thread safe.
+RPMALLOC_EXPORT rpmalloc_heap_t *rpmalloc_heap_acquire(void);
+
+//! Release a heap (does NOT free the memory allocated by the heap, use
+//! rpmalloc_heap_free_all before destroying the heap).
+// Releasing a heap will enable it to be reused by other threads. Safe to pass
+// a null pointer.
+RPMALLOC_EXPORT void rpmalloc_heap_release(rpmalloc_heap_t *heap);
+
+//! Allocate a memory block of at least the given size using the given heap.
+RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *
+rpmalloc_heap_alloc(rpmalloc_heap_t *heap, size_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(2);
+
+//! Allocate a memory block of at least the given size using the given heap. The
+//! returned
+// block will have the requested alignment. Alignment must be a power of two
+// and a multiple of sizeof(void*), and should ideally be less than memory page
+// size. A caveat of rpmalloc internals is that this must also be strictly less
+// than the span size (default 64KiB).
+RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *
+rpmalloc_heap_aligned_alloc(rpmalloc_heap_t *heap, size_t alignment,
+ size_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(3);
+
+//! Allocate a memory block of at least the given size using the given heap and
+//! zero initialize it.
+RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *
+rpmalloc_heap_calloc(rpmalloc_heap_t *heap, size_t num,
+ size_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE2(2, 3);
+
+//! Allocate a memory block of at least the given size using the given heap and
+//! zero initialize it. The returned
+// block will have the requested alignment. Alignment must either be zero, or a
+// power of two and a multiple of sizeof(void*), and should ideally be less
+// than memory page size. A caveat of rpmalloc internals is that this must also
+// be strictly less than the span size (default 64KiB).
+RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *
+rpmalloc_heap_aligned_calloc(rpmalloc_heap_t *heap, size_t alignment,
+ size_t num, size_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE2(2, 3);
+
+//! Reallocate the given block to at least the given size. The memory block MUST
+//! be allocated
+// by the same heap given to this function.
+RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *
+rpmalloc_heap_realloc(rpmalloc_heap_t *heap, void *ptr, size_t size,
+ unsigned int flags) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(3);
+
+//! Reallocate the given block to at least the given size. The memory block MUST
+//! be allocated
+// by the same heap given to this function. The returned block will have the
+// requested alignment. Alignment must be either zero, or a power of two and a
+// multiple of sizeof(void*), and should ideally be less than memory page size.
+// A caveat of rpmalloc internals is that this must also be strictly less than
+// the span size (default 64KiB).
+RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *rpmalloc_heap_aligned_realloc(
+ rpmalloc_heap_t *heap, void *ptr, size_t alignment, size_t size,
+ unsigned int flags) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(4);
+
+//! Free the given memory block from the given heap. The memory block MUST be
+//! allocated
+// by the same heap given to this function.
+RPMALLOC_EXPORT void rpmalloc_heap_free(rpmalloc_heap_t *heap, void *ptr);
+
+//! Free all memory allocated by the heap
+RPMALLOC_EXPORT void rpmalloc_heap_free_all(rpmalloc_heap_t *heap);
+
+//! Set the given heap as the current heap for the calling thread. A heap MUST
+//! only be current heap
+// for a single thread, a heap can never be shared between multiple threads.
+// The previous current heap for the calling thread is released to be reused by
+// other threads.
+RPMALLOC_EXPORT void rpmalloc_heap_thread_set_current(rpmalloc_heap_t *heap);
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/llvm/lib/Support/rpmalloc/rpnew.h b/llvm/lib/Support/rpmalloc/rpnew.h
new file mode 100644
index 0000000000000..51786ade5a3ea
--- /dev/null
+++ b/llvm/lib/Support/rpmalloc/rpnew.h
@@ -0,0 +1,109 @@
+//===---------------------------- rpnew.h ------------------*- C
+//-*-===========//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef __cplusplus
+
+#include <new>
+#include <rpmalloc.h>
+
+#ifdef _WIN32
+
+extern void __CRTDECL operator delete(void *p) noexcept { rpfree(p); }
+
+extern void __CRTDECL operator delete[](void *p) noexcept { rpfree(p); }
+
+extern void *__CRTDECL operator new(std::size_t size) noexcept(false) {
+ return rpmalloc(size);
+}
+
+extern void *__CRTDECL operator new[](std::size_t size) noexcept(false) {
+ return rpmalloc(size);
+}
+
+extern void *__CRTDECL operator new(std::size_t size,
+ const std::nothrow_t &tag) noexcept {
+ (void)sizeof(tag);
+ return rpmalloc(size);
+}
+
+extern void *__CRTDECL operator new[](std::size_t size,
+ const std::nothrow_t &tag) noexcept {
+ (void)sizeof(tag);
+ return rpmalloc(size);
+}
+
+#if (__cplusplus >= 201402L || _MSC_VER >= 1916)
+
+extern void __CRTDECL operator delete(void *p, std::size_t size) noexcept {
+ (void)sizeof(size);
+ rpfree(p);
+}
+
+extern void __CRTDECL operator delete[](void *p, std::size_t size) noexcept {
+ (void)sizeof(size);
+ rpfree(p);
+}
+
+#endif
+
+#if (__cplusplus > 201402L || defined(__cpp_aligned_new))
+
+extern void __CRTDECL operator delete(void *p,
+ std::align_val_t align) noexcept {
+ (void)sizeof(align);
+ rpfree(p);
+}
+
+extern void __CRTDECL operator delete[](void *p,
+ std::align_val_t align) noexcept {
+ (void)sizeof(align);
+ rpfree(p);
+}
+
+extern void __CRTDECL operator delete(void *p, std::size_t size,
+ std::align_val_t align) noexcept {
+ (void)sizeof(size);
+ (void)sizeof(align);
+ rpfree(p);
+}
+
+extern void __CRTDECL operator delete[](void *p, std::size_t size,
+ std::align_val_t align) noexcept {
+ (void)sizeof(size);
+ (void)sizeof(align);
+ rpfree(p);
+}
+
+extern void *__CRTDECL operator new(std::size_t size,
+ std::align_val_t align) noexcept(false) {
+ return rpaligned_alloc((size_t)align, size);
+}
+
+extern void *__CRTDECL operator new[](std::size_t size,
+ std::align_val_t align) noexcept(false) {
+ return rpaligned_alloc((size_t)align, size);
+}
+
+extern void *__CRTDECL operator new(std::size_t size, std::align_val_t align,
+ const std::nothrow_t &tag) noexcept {
+ (void)sizeof(tag);
+ return rpaligned_alloc((size_t)align, size);
+}
+
+extern void *__CRTDECL operator new[](std::size_t size, std::align_val_t align,
+ const std::nothrow_t &tag) noexcept {
+ (void)sizeof(tag);
+ return rpaligned_alloc((size_t)align, size);
+}
+
+#endif
+
+#endif
+
+#endif
diff --git a/llvm/utils/release/build_llvm_release.bat b/llvm/utils/release/build_llvm_release.bat
index dc63fdac1e4ad..4ffded06666bb 100755
--- a/llvm/utils/release/build_llvm_release.bat
+++ b/llvm/utils/release/build_llvm_release.bat
@@ -163,6 +163,8 @@ set common_cmake_flags=^
-DCLANG_ENABLE_LIBXML2=OFF ^
-DCMAKE_C_FLAGS="%common_compiler_flags%" ^
-DCMAKE_CXX_FLAGS="%common_compiler_flags%" ^
+ -DLLVM_ENABLE_RPMALLOC=ON ^
+ -DLLVM_ENABLE_PDB=ON ^
-DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra;lld;compiler-rt;lldb;openmp"
set cmake_profile_flags=""
@@ -412,6 +414,7 @@ cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=install ^
-DLIBXML2_WITH_THREADS=ON -DLIBXML2_WITH_THREAD_ALLOC=OFF -DLIBXML2_WITH_TREE=ON ^
-DLIBXML2_WITH_VALID=OFF -DLIBXML2_WITH_WRITER=OFF -DLIBXML2_WITH_XINCLUDE=OFF ^
-DLIBXML2_WITH_XPATH=OFF -DLIBXML2_WITH_XPTR=OFF -DLIBXML2_WITH_ZLIB=OFF ^
+ -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded ^
../../libxml2-v2.9.12 || exit /b 1
ninja install || exit /b 1
set libxmldir=%cd%\install
>From 96db69e2e640dd071ec467f2c88f9a2bfb0f4767 Mon Sep 17 00:00:00 2001
From: Alexandre Ganea <aganea at havenstudios.com>
Date: Sun, 5 May 2024 20:30:51 -0400
Subject: [PATCH 2/8] Fix building with latest cmake
---
llvm/utils/release/build_llvm_release.bat | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/utils/release/build_llvm_release.bat b/llvm/utils/release/build_llvm_release.bat
index 4ffded06666bb..787b51bfa53f8 100755
--- a/llvm/utils/release/build_llvm_release.bat
+++ b/llvm/utils/release/build_llvm_release.bat
@@ -252,7 +252,7 @@ set cmake_flags=^
-DLLDB_TEST_COMPILER=%stage0_bin_dir%/clang.exe ^
-DPYTHON_HOME=%PYTHONHOME% ^
-DPython3_ROOT_DIR=%PYTHONHOME% ^
- -DLIBXML2_INCLUDE_DIRS=%libxmldir%/include/libxml2 ^
+ -DLIBXML2_INCLUDE_DIR=%libxmldir%/include/libxml2 ^
-DLIBXML2_LIBRARIES=%libxmldir%/lib/libxml2s.lib
cmake -GNinja %cmake_flags% %llvm_src%\llvm || exit /b 1
>From cb77fc76662f99279efa46a409b949531baf69a9 Mon Sep 17 00:00:00 2001
From: Alexandre Ganea <aganea at havenstudios.com>
Date: Mon, 13 May 2024 08:41:06 -0400
Subject: [PATCH 3/8] Use rpmalloc 1.4.5 and fix clang-format
---
llvm/lib/Support/rpmalloc/README.md | 60 +++-
llvm/lib/Support/rpmalloc/malloc.c | 413 ++++++++++++++++-----------
llvm/lib/Support/rpmalloc/rpmalloc.c | 161 +++++++++--
llvm/lib/Support/rpmalloc/rpmalloc.h | 9 +-
llvm/lib/Support/rpmalloc/rpnew.h | 22 +-
5 files changed, 450 insertions(+), 215 deletions(-)
diff --git a/llvm/lib/Support/rpmalloc/README.md b/llvm/lib/Support/rpmalloc/README.md
index 5fba25ebc8095..6173876a0f632 100644
--- a/llvm/lib/Support/rpmalloc/README.md
+++ b/llvm/lib/Support/rpmalloc/README.md
@@ -1,7 +1,5 @@
# rpmalloc - General Purpose Memory Allocator
-This library provides a public domain cross platform lock free thread caching 16-byte aligned memory allocator implemented in C. The latest source code is always available at https://github.com/mjansson/rpmalloc
-
-Created by Mattias Jansson ([@maniccoder](https://twitter.com/maniccoder)) - Discord server for discussions at https://discord.gg/M8BwTQrt6c
+This library provides a cross platform lock free thread caching 16-byte aligned memory allocator implemented in C.
Platforms currently supported:
@@ -54,7 +52,7 @@ __rpmalloc_config__: Get the current runtime configuration of the allocator
Then simply use the __rpmalloc__/__rpfree__ and the other malloc style replacement functions. Remember all allocations are 16-byte aligned, so no need to call the explicit rpmemalign/rpaligned_alloc/rpposix_memalign functions unless you need greater alignment, they are simply wrappers to make it easier to replace in existing code.
-If you wish to override the standard library malloc family of functions and have automatic initialization/finalization of process and threads, define __ENABLE_OVERRIDE__ to non-zero which will include the `malloc.c` file in compilation of __rpmalloc.c__. The list of libc entry points replaced may not be complete, use libc replacement only as a convenience for testing the library on an existing code base, not a final solution.
+If you wish to override the standard library malloc family of functions and have automatic initialization/finalization of process and threads, define __ENABLE_OVERRIDE__ to non-zero which will include the `malloc.c` file in compilation of __rpmalloc.c__, and then rebuild the library or your project where you added the rpmalloc source. If you compile rpmalloc as a separate library you must make the linker use the override symbols from the library by referencing at least one symbol. The easiest way is to simply include `rpmalloc.h` in at least one source file and call `rpmalloc_linker_reference` somewhere - it's a dummy empty function. On Windows platforms and C++ overrides you have to `#include <rpnew.h>` in at least one source file and also manually handle the initialize/finalize of the process and all threads. The list of libc entry points replaced may not be complete, use libc/stdc++ replacement only as a convenience for testing the library on an existing code base, not a final solution.
For explicit first class heaps, see the __rpmalloc_heap_*__ API under [first class heaps](#first-class-heaps) section, requiring __RPMALLOC_FIRST_CLASS_HEAPS__ tp be defined to 1.
@@ -168,6 +166,54 @@ To support global scope data doing dynamic allocation/deallocation such as C++ o
# License
-As part of the LLVM codebase, rpmalloc has be re-licenced under the the Apache License v2.0 with LLVM Exceptions.
-See https://llvm.org/LICENSE.txt for license information.
-SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <http://unlicense.org>
+
+
+You can also use this software under the MIT license if public domain is
+not recognized in your country
+
+
+The MIT License (MIT)
+
+Copyright (c) 2017 Mattias Jansson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/llvm/lib/Support/rpmalloc/malloc.c b/llvm/lib/Support/rpmalloc/malloc.c
index d4f99e5fe1d04..498aa2782586c 100644
--- a/llvm/lib/Support/rpmalloc/malloc.c
+++ b/llvm/lib/Support/rpmalloc/malloc.c
@@ -1,4 +1,4 @@
-//===------------------------ malloc.c -----------------*- C -*-=============//
+//===------------------------ malloc.c ------------------*- C -*-=============//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -9,8 +9,9 @@
// This library provides a cross-platform lock free thread caching malloc
// implementation in C11.
//
+//
// This file provides overrides for the standard library malloc entry points for
-// C and new/delete operators for C++. It also provides automatic
+// C and new/delete operators for C++ It also provides automatic
// initialization/finalization of process and threads
//
//===----------------------------------------------------------------------===//
@@ -31,7 +32,7 @@ _Static_assert(sizeof(void *) == 4, "Data type size mismatch");
#endif
#endif
-#if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__)
+#if (defined(__GNUC__) || defined(__clang__))
#pragma GCC visibility push(default)
#endif
@@ -39,7 +40,7 @@ _Static_assert(sizeof(void *) == 4, "Data type size mismatch");
#define USE_INTERPOSE 0
#define USE_ALIAS 0
-#if defined(__APPLE__) && ENABLE_PRELOAD
+#if defined(__APPLE__)
#undef USE_INTERPOSE
#define USE_INTERPOSE 1
@@ -48,8 +49,7 @@ typedef struct interpose_t {
void *orig_func;
} interpose_t;
-#define MAC_INTERPOSE_PAIR(newf, oldf) \
- { (void *)newf, (void *)oldf }
+#define MAC_INTERPOSE_PAIR(newf, oldf) {(void *)newf, (void *)oldf}
#define MAC_INTERPOSE_SINGLE(newf, oldf) \
__attribute__((used)) static const interpose_t macinterpose##newf##oldf \
__attribute__((section("__DATA, __interpose"))) = \
@@ -57,7 +57,7 @@ typedef struct interpose_t {
#endif
-#if !defined(_WIN32) && !USE_INTERPOSE
+#if !defined(_WIN32) && !defined(__APPLE__)
#undef USE_IMPLEMENT
#undef USE_ALIAS
#define USE_IMPLEMENT 0
@@ -116,6 +116,27 @@ extern inline size_t RPMALLOC_CDECL malloc_size(void *ptr) {
return rpmalloc_usable_size(ptr);
}
+#ifdef _WIN32
+extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL _malloc_base(size_t size) {
+ return rpmalloc(size);
+}
+extern inline void RPMALLOC_CDECL _free_base(void *ptr) { rpfree(ptr); }
+extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL _calloc_base(size_t count,
+ size_t size) {
+ return rpcalloc(count, size);
+}
+extern inline size_t RPMALLOC_CDECL _msize(void *ptr) {
+ return rpmalloc_usable_size(ptr);
+}
+extern inline size_t RPMALLOC_CDECL _msize_base(void *ptr) {
+ return rpmalloc_usable_size(ptr);
+}
+extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL
+_realloc_base(void *ptr, size_t size) {
+ return rprealloc(ptr, size);
+}
+#endif
+
#ifdef _WIN32
// For Windows, #include <rpnew.h> in one source file to get the C++ operator
// overrides implemented in your module
@@ -123,85 +144,86 @@ extern inline size_t RPMALLOC_CDECL malloc_size(void *ptr) {
// Overload the C++ operators using the mangled names
// (https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling) operators
// delete and delete[]
+#define RPDEFVIS __attribute__((visibility("default")))
extern void _ZdlPv(void *p);
-void _ZdlPv(void *p) { rpfree(p); }
+void RPDEFVIS _ZdlPv(void *p) { rpfree(p); }
extern void _ZdaPv(void *p);
-void _ZdaPv(void *p) { rpfree(p); }
+void RPDEFVIS _ZdaPv(void *p) { rpfree(p); }
#if ARCH_64BIT
// 64-bit operators new and new[], normal and aligned
extern void *_Znwm(uint64_t size);
-void *_Znwm(uint64_t size) { return rpmalloc(size); }
+void *RPDEFVIS _Znwm(uint64_t size) { return rpmalloc(size); }
extern void *_Znam(uint64_t size);
-void *_Znam(uint64_t size) { return rpmalloc(size); }
+void *RPDEFVIS _Znam(uint64_t size) { return rpmalloc(size); }
extern void *_Znwmm(uint64_t size, uint64_t align);
-void *_Znwmm(uint64_t size, uint64_t align) {
+void *RPDEFVIS _Znwmm(uint64_t size, uint64_t align) {
return rpaligned_alloc(align, size);
}
extern void *_Znamm(uint64_t size, uint64_t align);
-void *_Znamm(uint64_t size, uint64_t align) {
+void *RPDEFVIS _Znamm(uint64_t size, uint64_t align) {
return rpaligned_alloc(align, size);
}
extern void *_ZnwmSt11align_val_t(uint64_t size, uint64_t align);
-void *_ZnwmSt11align_val_t(uint64_t size, uint64_t align) {
+void *RPDEFVIS _ZnwmSt11align_val_t(uint64_t size, uint64_t align) {
return rpaligned_alloc(align, size);
}
extern void *_ZnamSt11align_val_t(uint64_t size, uint64_t align);
-void *_ZnamSt11align_val_t(uint64_t size, uint64_t align) {
+void *RPDEFVIS _ZnamSt11align_val_t(uint64_t size, uint64_t align) {
return rpaligned_alloc(align, size);
}
extern void *_ZnwmRKSt9nothrow_t(uint64_t size, rp_nothrow_t t);
-void *_ZnwmRKSt9nothrow_t(uint64_t size, rp_nothrow_t t) {
+void *RPDEFVIS _ZnwmRKSt9nothrow_t(uint64_t size, rp_nothrow_t t) {
(void)sizeof(t);
return rpmalloc(size);
}
extern void *_ZnamRKSt9nothrow_t(uint64_t size, rp_nothrow_t t);
-void *_ZnamRKSt9nothrow_t(uint64_t size, rp_nothrow_t t) {
+void *RPDEFVIS _ZnamRKSt9nothrow_t(uint64_t size, rp_nothrow_t t) {
(void)sizeof(t);
return rpmalloc(size);
}
extern void *_ZnwmSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
rp_nothrow_t t);
-void *_ZnwmSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
- rp_nothrow_t t) {
+void *RPDEFVIS _ZnwmSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
+ rp_nothrow_t t) {
(void)sizeof(t);
return rpaligned_alloc(align, size);
}
extern void *_ZnamSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
rp_nothrow_t t);
-void *_ZnamSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
- rp_nothrow_t t) {
+void *RPDEFVIS _ZnamSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
+ rp_nothrow_t t) {
(void)sizeof(t);
return rpaligned_alloc(align, size);
}
// 64-bit operators sized delete and delete[], normal and aligned
extern void _ZdlPvm(void *p, uint64_t size);
-void _ZdlPvm(void *p, uint64_t size) {
+void RPDEFVIS _ZdlPvm(void *p, uint64_t size) {
rpfree(p);
(void)sizeof(size);
}
extern void _ZdaPvm(void *p, uint64_t size);
-void _ZdaPvm(void *p, uint64_t size) {
+void RPDEFVIS _ZdaPvm(void *p, uint64_t size) {
rpfree(p);
(void)sizeof(size);
}
extern void _ZdlPvSt11align_val_t(void *p, uint64_t align);
-void _ZdlPvSt11align_val_t(void *p, uint64_t align) {
+void RPDEFVIS _ZdlPvSt11align_val_t(void *p, uint64_t align) {
rpfree(p);
(void)sizeof(align);
}
extern void _ZdaPvSt11align_val_t(void *p, uint64_t align);
-void _ZdaPvSt11align_val_t(void *p, uint64_t align) {
+void RPDEFVIS _ZdaPvSt11align_val_t(void *p, uint64_t align) {
rpfree(p);
(void)sizeof(align);
}
extern void _ZdlPvmSt11align_val_t(void *p, uint64_t size, uint64_t align);
-void _ZdlPvmSt11align_val_t(void *p, uint64_t size, uint64_t align) {
+void RPDEFVIS _ZdlPvmSt11align_val_t(void *p, uint64_t size, uint64_t align) {
rpfree(p);
(void)sizeof(size);
(void)sizeof(align);
}
extern void _ZdaPvmSt11align_val_t(void *p, uint64_t size, uint64_t align);
-void _ZdaPvmSt11align_val_t(void *p, uint64_t size, uint64_t align) {
+void RPDEFVIS _ZdaPvmSt11align_val_t(void *p, uint64_t size, uint64_t align) {
rpfree(p);
(void)sizeof(size);
(void)sizeof(align);
@@ -209,84 +231,113 @@ void _ZdaPvmSt11align_val_t(void *p, uint64_t size, uint64_t align) {
#else
// 32-bit operators new and new[], normal and aligned
extern void *_Znwj(uint32_t size);
-void *_Znwj(uint32_t size) { return rpmalloc(size); }
+void *RPDEFVIS _Znwj(uint32_t size) { return rpmalloc(size); }
extern void *_Znaj(uint32_t size);
-void *_Znaj(uint32_t size) { return rpmalloc(size); }
+void *RPDEFVIS _Znaj(uint32_t size) { return rpmalloc(size); }
extern void *_Znwjj(uint32_t size, uint32_t align);
-void *_Znwjj(uint32_t size, uint32_t align) {
+void *RPDEFVIS _Znwjj(uint32_t size, uint32_t align) {
return rpaligned_alloc(align, size);
}
extern void *_Znajj(uint32_t size, uint32_t align);
-void *_Znajj(uint32_t size, uint32_t align) {
+void *RPDEFVIS _Znajj(uint32_t size, uint32_t align) {
return rpaligned_alloc(align, size);
}
extern void *_ZnwjSt11align_val_t(size_t size, size_t align);
-void *_ZnwjSt11align_val_t(size_t size, size_t align) {
+void *RPDEFVIS _ZnwjSt11align_val_t(size_t size, size_t align) {
return rpaligned_alloc(align, size);
}
extern void *_ZnajSt11align_val_t(size_t size, size_t align);
-void *_ZnajSt11align_val_t(size_t size, size_t align) {
+void *RPDEFVIS _ZnajSt11align_val_t(size_t size, size_t align) {
return rpaligned_alloc(align, size);
}
extern void *_ZnwjRKSt9nothrow_t(size_t size, rp_nothrow_t t);
-void *_ZnwjRKSt9nothrow_t(size_t size, rp_nothrow_t t) {
+void *RPDEFVIS _ZnwjRKSt9nothrow_t(size_t size, rp_nothrow_t t) {
(void)sizeof(t);
return rpmalloc(size);
}
extern void *_ZnajRKSt9nothrow_t(size_t size, rp_nothrow_t t);
-void *_ZnajRKSt9nothrow_t(size_t size, rp_nothrow_t t) {
+void *RPDEFVIS _ZnajRKSt9nothrow_t(size_t size, rp_nothrow_t t) {
(void)sizeof(t);
return rpmalloc(size);
}
extern void *_ZnwjSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
rp_nothrow_t t);
-void *_ZnwjSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
- rp_nothrow_t t) {
+void *RPDEFVIS _ZnwjSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
+ rp_nothrow_t t) {
(void)sizeof(t);
return rpaligned_alloc(align, size);
}
extern void *_ZnajSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
rp_nothrow_t t);
-void *_ZnajSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
- rp_nothrow_t t) {
+void *RPDEFVIS _ZnajSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
+ rp_nothrow_t t) {
(void)sizeof(t);
return rpaligned_alloc(align, size);
}
// 32-bit operators sized delete and delete[], normal and aligned
extern void _ZdlPvj(void *p, uint64_t size);
-void _ZdlPvj(void *p, uint64_t size) {
+void RPDEFVIS _ZdlPvj(void *p, uint64_t size) {
rpfree(p);
(void)sizeof(size);
}
extern void _ZdaPvj(void *p, uint64_t size);
-void _ZdaPvj(void *p, uint64_t size) {
+void RPDEFVIS _ZdaPvj(void *p, uint64_t size) {
rpfree(p);
(void)sizeof(size);
}
extern void _ZdlPvSt11align_val_t(void *p, uint32_t align);
-void _ZdlPvSt11align_val_t(void *p, uint64_t a) {
+void RPDEFVIS _ZdlPvSt11align_val_t(void *p, uint64_t a) {
rpfree(p);
(void)sizeof(align);
}
extern void _ZdaPvSt11align_val_t(void *p, uint32_t align);
-void _ZdaPvSt11align_val_t(void *p, uint64_t a) {
+void RPDEFVIS _ZdaPvSt11align_val_t(void *p, uint64_t a) {
rpfree(p);
(void)sizeof(align);
}
extern void _ZdlPvjSt11align_val_t(void *p, uint32_t size, uint32_t align);
-void _ZdlPvjSt11align_val_t(void *p, uint64_t size, uint64_t align) {
+void RPDEFVIS _ZdlPvjSt11align_val_t(void *p, uint64_t size, uint64_t align) {
rpfree(p);
(void)sizeof(size);
(void)sizeof(a);
}
extern void _ZdaPvjSt11align_val_t(void *p, uint32_t size, uint32_t align);
-void _ZdaPvjSt11align_val_t(void *p, uint64_t size, uint64_t align) {
+void RPDEFVIS _ZdaPvjSt11align_val_t(void *p, uint64_t size, uint64_t align) {
rpfree(p);
(void)sizeof(size);
(void)sizeof(a);
}
#endif
#endif
+#endif
+
+#if USE_INTERPOSE || USE_ALIAS
+
+static void *rpmalloc_nothrow(size_t size, rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpmalloc(size);
+}
+static void *rpaligned_alloc_reverse(size_t size, size_t align) {
+ return rpaligned_alloc(align, size);
+}
+static void *rpaligned_alloc_reverse_nothrow(size_t size, size_t align,
+ rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpaligned_alloc(align, size);
+}
+static void rpfree_size(void *p, size_t size) {
+ (void)sizeof(size);
+ rpfree(p);
+}
+static void rpfree_aligned(void *p, size_t align) {
+ (void)sizeof(align);
+ rpfree(p);
+}
+static void rpfree_size_aligned(void *p, size_t size, size_t align) {
+ (void)sizeof(size);
+ (void)sizeof(align);
+ rpfree(p);
+}
#endif
@@ -297,8 +348,25 @@ __attribute__((used)) static const interpose_t macinterpose_malloc[]
// new and new[]
MAC_INTERPOSE_PAIR(rpmalloc, _Znwm),
MAC_INTERPOSE_PAIR(rpmalloc, _Znam),
+ MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _Znwmm),
+ MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _Znamm),
+ MAC_INTERPOSE_PAIR(rpmalloc_nothrow, _ZnwmRKSt9nothrow_t),
+ MAC_INTERPOSE_PAIR(rpmalloc_nothrow, _ZnamRKSt9nothrow_t),
+ MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _ZnwmSt11align_val_t),
+ MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _ZnamSt11align_val_t),
+ MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse_nothrow,
+ _ZnwmSt11align_val_tRKSt9nothrow_t),
+ MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse_nothrow,
+ _ZnamSt11align_val_tRKSt9nothrow_t),
// delete and delete[]
MAC_INTERPOSE_PAIR(rpfree, _ZdlPv), MAC_INTERPOSE_PAIR(rpfree, _ZdaPv),
+ MAC_INTERPOSE_PAIR(rpfree_size, _ZdlPvm),
+ MAC_INTERPOSE_PAIR(rpfree_size, _ZdaPvm),
+ MAC_INTERPOSE_PAIR(rpfree_aligned, _ZdlPvSt11align_val_t),
+ MAC_INTERPOSE_PAIR(rpfree_aligned, _ZdaPvSt11align_val_t),
+ MAC_INTERPOSE_PAIR(rpfree_size_aligned, _ZdlPvmSt11align_val_t),
+ MAC_INTERPOSE_PAIR(rpfree_size_aligned, _ZdaPvmSt11align_val_t),
+ // libc entry points
MAC_INTERPOSE_PAIR(rpmalloc, malloc),
MAC_INTERPOSE_PAIR(rpmalloc, calloc),
MAC_INTERPOSE_PAIR(rprealloc, realloc),
@@ -322,123 +390,106 @@ __attribute__((used)) static const interpose_t macinterpose_malloc[]
// (https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling)
// operators delete and delete[]
-void _ZdlPv(void *p) RPALIAS(rpfree) void _ZdaPv(void *p)
- RPALIAS(rpfree) extern inline void _ZdlPvm(void *p, size_t n) {
- rpfree(p);
- (void)sizeof(n);
-}
-extern inline void _ZdaPvm(void *p, size_t n) {
- rpfree(p);
- (void)sizeof(n);
-}
-extern inline void _ZdlPvSt11align_val_t(void *p, size_t a) {
- rpfree(p);
- (void)sizeof(a);
-}
-extern inline void _ZdaPvSt11align_val_t(void *p, size_t a) {
- rpfree(p);
- (void)sizeof(a);
-}
-extern inline void _ZdlPvmSt11align_val_t(void *p, size_t n, size_t a) {
- rpfree(p);
- (void)sizeof(n);
- (void)sizeof(a);
-}
-extern inline void _ZdaPvmSt11align_val_t(void *p, size_t n, size_t a) {
- rpfree(p);
- (void)sizeof(n);
- (void)sizeof(a);
-}
+void _ZdlPv(void *p) RPALIAS(rpfree) void _ZdaPv(void *p) RPALIAS(rpfree)
#if ARCH_64BIT
-// 64-bit operators new and new[], normal and aligned
-void *_Znwm(uint64_t size) RPALIAS(rpmalloc) void *_Znam(uint64_t size)
- RPALIAS(rpmalloc) extern inline void *_Znwmm(uint64_t size,
- uint64_t align) {
- return rpaligned_alloc(align, size);
-}
-extern inline void *_Znamm(uint64_t size, uint64_t align) {
- return rpaligned_alloc(align, size);
-}
-extern inline void *_ZnwmSt11align_val_t(size_t size, size_t align) {
- return rpaligned_alloc(align, size);
-}
-extern inline void *_ZnamSt11align_val_t(size_t size, size_t align) {
- return rpaligned_alloc(align, size);
-}
-extern inline void *_ZnwmRKSt9nothrow_t(size_t size, rp_nothrow_t t) {
- (void)sizeof(t);
- return rpmalloc(size);
-}
-extern inline void *_ZnamRKSt9nothrow_t(size_t size, rp_nothrow_t t) {
- (void)sizeof(t);
- return rpmalloc(size);
-}
-extern inline void *
-_ZnwmSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t) {
- (void)sizeof(t);
- return rpaligned_alloc(align, size);
-}
-extern inline void *
-_ZnamSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t) {
- (void)sizeof(t);
- return rpaligned_alloc(align, size);
-}
+ // 64-bit operators new and new[], normal and aligned
+ void *_Znwm(uint64_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(1)
+ RPALIAS(rpmalloc) void *_Znam(uint64_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(1) RPALIAS(rpmalloc) void *_Znwmm(uint64_t size,
+ uint64_t align)
+ RPALIAS(rpaligned_alloc_reverse) void *_Znamm(uint64_t size,
+ uint64_t align)
+ RPALIAS(rpaligned_alloc_reverse) void *_ZnwmSt11align_val_t(
+ size_t size, size_t align)
+ RPALIAS(rpaligned_alloc_reverse) void *_ZnamSt11align_val_t(
+ size_t size, size_t align)
+ RPALIAS(rpaligned_alloc_reverse) void *_ZnwmRKSt9nothrow_t(
+ size_t size, rp_nothrow_t t)
+ RPALIAS(rpmalloc_nothrow) void *_ZnamRKSt9nothrow_t(
+ size_t size,
+ rp_nothrow_t t) RPALIAS(rpmalloc_nothrow) void
+ *_ZnwmSt11align_val_tRKSt9nothrow_t(size_t size,
+ size_t align,
+ rp_nothrow_t t)
+ RPALIAS(rpaligned_alloc_reverse_nothrow) void
+ *_ZnamSt11align_val_tRKSt9nothrow_t(
+ size_t size, size_t align,
+ rp_nothrow_t t)
+ RPALIAS(rpaligned_alloc_reverse_nothrow)
+ // 64-bit operators delete and delete[], sized and aligned
+ void _ZdlPvm(void *p, size_t n) RPALIAS(rpfree_size) void _ZdaPvm(void *p,
+ size_t n)
+ RPALIAS(rpfree_size) void _ZdlPvSt11align_val_t(void *p, size_t a)
+ RPALIAS(rpfree_aligned) void _ZdaPvSt11align_val_t(void *p,
+ size_t a)
+ RPALIAS(rpfree_aligned) void _ZdlPvmSt11align_val_t(void *p,
+ size_t n,
+ size_t a)
+ RPALIAS(rpfree_size_aligned) void _ZdaPvmSt11align_val_t(
+ void *p, size_t n, size_t a)
+ RPALIAS(rpfree_size_aligned)
#else
-// 32-bit operators new and new[], normal and aligned
-void *_Znwj(uint32_t size) RPALIAS(rpmalloc) void *_Znaj(uint32_t size)
- RPALIAS(rpmalloc) extern inline void *_Znwjj(uint32_t size,
- uint32_t align) {
- return rpaligned_alloc(align, size);
-}
-extern inline void *_Znajj(uint32_t size, uint32_t align) {
- return rpaligned_alloc(align, size);
-}
-extern inline void *_ZnwjSt11align_val_t(size_t size, size_t align) {
- return rpaligned_alloc(align, size);
-}
-extern inline void *_ZnajSt11align_val_t(size_t size, size_t align) {
- return rpaligned_alloc(align, size);
-}
-extern inline void *_ZnwjRKSt9nothrow_t(size_t size, rp_nothrow_t t) {
- (void)sizeof(t);
- return rpmalloc(size);
-}
-extern inline void *_ZnajRKSt9nothrow_t(size_t size, rp_nothrow_t t) {
- (void)sizeof(t);
- return rpmalloc(size);
-}
-extern inline void *
-_ZnwjSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t) {
- (void)sizeof(t);
- return rpaligned_alloc(align, size);
-}
-extern inline void *
-_ZnajSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t) {
- (void)sizeof(t);
- return rpaligned_alloc(align, size);
-}
+ // 32-bit operators new and new[], normal and aligned
+ void *_Znwj(uint32_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(1)
+ RPALIAS(rpmalloc) void *_Znaj(uint32_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(1) RPALIAS(rpmalloc) void *_Znwjj(uint32_t size,
+ uint32_t align)
+ RPALIAS(rpaligned_alloc_reverse) void *_Znajj(uint32_t size,
+ uint32_t align)
+ RPALIAS(rpaligned_alloc_reverse) void *_ZnwjSt11align_val_t(
+ size_t size, size_t align)
+ RPALIAS(rpaligned_alloc_reverse) void *_ZnajSt11align_val_t(
+ size_t size, size_t align)
+ RPALIAS(rpaligned_alloc_reverse) void *_ZnwjRKSt9nothrow_t(
+ size_t size, rp_nothrow_t t)
+ RPALIAS(rpmalloc_nothrow) void *_ZnajRKSt9nothrow_t(
+ size_t size,
+ rp_nothrow_t t) RPALIAS(rpmalloc_nothrow) void
+ *_ZnwjSt11align_val_tRKSt9nothrow_t(size_t size,
+ size_t align,
+ rp_nothrow_t t)
+ RPALIAS(rpaligned_alloc_reverse_nothrow) void
+ *_ZnajSt11align_val_tRKSt9nothrow_t(
+ size_t size, size_t align,
+ rp_nothrow_t t)
+ RPALIAS(rpaligned_alloc_reverse_nothrow)
+ // 32-bit operators delete and delete[], sized and aligned
+ void _ZdlPvj(void *p, size_t n) RPALIAS(rpfree_size) void _ZdaPvj(void *p,
+ size_t n)
+ RPALIAS(rpfree_size) void _ZdlPvSt11align_val_t(void *p, size_t a)
+ RPALIAS(rpfree_aligned) void _ZdaPvSt11align_val_t(void *p,
+ size_t a)
+ RPALIAS(rpfree_aligned) void _ZdlPvjSt11align_val_t(void *p,
+ size_t n,
+ size_t a)
+ RPALIAS(rpfree_size_aligned) void _ZdaPvjSt11align_val_t(
+ void *p, size_t n, size_t a)
+ RPALIAS(rpfree_size_aligned)
#endif
-void *malloc(size_t size)
- RPALIAS(rpmalloc) void *calloc(size_t count, size_t size)
- RPALIAS(rpcalloc) void *realloc(void *ptr, size_t size)
- RPALIAS(rprealloc) void *reallocf(void *ptr, size_t size)
- RPALIAS(rprealloc) void *aligned_alloc(size_t alignment,
- size_t size)
- RPALIAS(rpaligned_alloc) void *memalign(size_t alignment,
- size_t size)
- RPALIAS(rpmemalign) int posix_memalign(void **memptr,
- size_t alignment,
- size_t size)
- RPALIAS(rpposix_memalign) void free(void *ptr)
- RPALIAS(rpfree) void cfree(void *ptr)
- RPALIAS(rpfree)
+ void *malloc(size_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(1)
+ RPALIAS(rpmalloc) void *calloc(size_t count, size_t size)
+ RPALIAS(rpcalloc) void *realloc(void *ptr, size_t size)
+ RPALIAS(rprealloc) void *reallocf(void *ptr, size_t size)
+ RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(2)
+ RPALIAS(rprealloc) void *aligned_alloc(size_t alignment, size_t size)
+ RPALIAS(rpaligned_alloc) void *memalign(
+ size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(2)
+ RPALIAS(rpmemalign) int posix_memalign(void **memptr, size_t alignment,
+ size_t size)
+ RPALIAS(rpposix_memalign) void free(void *ptr)
+ RPALIAS(rpfree) void cfree(void *ptr) RPALIAS(rpfree)
#if defined(__ANDROID__) || defined(__FreeBSD__)
- size_t
+ size_t
malloc_usable_size(const void *ptr) RPALIAS(rpmalloc_usable_size)
#else
- size_t
+ size_t
malloc_usable_size(void *ptr) RPALIAS(rpmalloc_usable_size)
#endif
size_t malloc_size(void *ptr) RPALIAS(rpmalloc_usable_size)
@@ -449,8 +500,9 @@ void *malloc(size_t size)
return _memory_page_size;
}
-extern inline void *RPMALLOC_CDECL reallocarray(void *ptr, size_t count,
- size_t size) {
+extern void *RPMALLOC_CDECL reallocarray(void *ptr, size_t count, size_t size);
+
+extern void *RPMALLOC_CDECL reallocarray(void *ptr, size_t count, size_t size) {
size_t total;
#if ENABLE_VALIDATE_ARGS
#ifdef _MSC_VER
@@ -499,7 +551,8 @@ extern inline void *RPMALLOC_CDECL pvalloc(size_t size) {
#if defined(BUILD_DYNAMIC_LINK) && BUILD_DYNAMIC_LINK
extern __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE instance,
- DWORD reason, LPVOID reserved);
+ DWORD reason,
+ LPVOID reserved);
extern __declspec(dllexport) BOOL WINAPI
DllMain(HINSTANCE instance, DWORD reason, LPVOID reserved) {
@@ -532,10 +585,21 @@ static void __attribute__((constructor)) initializer(void) {
#elif defined(_MSC_VER)
+static int _global_rpmalloc_xib(void) {
+ _global_rpmalloc_init();
+ return 0;
+}
+
#pragma section(".CRT$XIB", read)
__declspec(allocate(".CRT$XIB")) void (*_rpmalloc_module_init)(void) =
- _global_rpmalloc_init;
-#pragma comment(linker, "/include:_rpmalloc_module_init")
+ _global_rpmalloc_xib;
+#if defined(_M_IX86) || defined(__i386__)
+#pragma comment(linker, "/include:" \
+ "__rpmalloc_module_init")
+#else
+#pragma comment(linker, "/include:" \
+ "_rpmalloc_module_init")
+#endif
#endif
@@ -629,19 +693,22 @@ int pthread_create(pthread_t *thread, const pthread_attr_t *attr,
#if defined(__GLIBC__) && defined(__linux__)
-void *__libc_malloc(size_t size)
- RPALIAS(rpmalloc) void *__libc_calloc(size_t count, size_t size)
- RPALIAS(rpcalloc) void *__libc_realloc(void *p, size_t size)
- RPALIAS(rprealloc) void __libc_free(void *p)
- RPALIAS(rpfree) void __libc_cfree(void *p)
- RPALIAS(rpfree) void *__libc_memalign(size_t align,
- size_t size)
- RPALIAS(rpmemalign) int __posix_memalign(void **p,
- size_t align,
- size_t size)
- RPALIAS(rpposix_memalign)
-
- extern void *__libc_valloc(size_t size);
+void *__libc_malloc(size_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(1)
+ RPALIAS(rpmalloc) void *__libc_calloc(size_t count, size_t size)
+ RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE2(1, 2)
+ RPALIAS(rpcalloc) void *__libc_realloc(void *p, size_t size)
+ RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(2) RPALIAS(rprealloc) void __libc_free(void *p)
+ RPALIAS(rpfree) void __libc_cfree(void *p)
+ RPALIAS(rpfree) void *__libc_memalign(size_t align, size_t size)
+ RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(2)
+ RPALIAS(rpmemalign) int __posix_memalign(void **p, size_t align,
+ size_t size)
+ RPALIAS(rpposix_memalign)
+
+ extern void *__libc_valloc(size_t size);
extern void *__libc_pvalloc(size_t size);
void *__libc_valloc(size_t size) { return valloc(size); }
@@ -652,6 +719,6 @@ void *__libc_pvalloc(size_t size) { return pvalloc(size); }
#endif
-#if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__)
+#if (defined(__GNUC__) || defined(__clang__))
#pragma GCC visibility pop
#endif
diff --git a/llvm/lib/Support/rpmalloc/rpmalloc.c b/llvm/lib/Support/rpmalloc/rpmalloc.c
index 0f9257563a104..53accc955a086 100644
--- a/llvm/lib/Support/rpmalloc/rpmalloc.c
+++ b/llvm/lib/Support/rpmalloc/rpmalloc.c
@@ -1,4 +1,5 @@
-//===-------------------------- rpmalloc.c ----------------*- C -*-===========//
+//===------------------------ rpmalloc.c -----------------*- C
+//-*-=============//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -25,11 +26,60 @@
#if __has_warning("-Wreserved-identifier")
#pragma clang diagnostic ignored "-Wreserved-identifier"
#endif
+#if __has_warning("-Wstatic-in-inline")
+#pragma clang diagnostic ignored "-Wstatic-in-inline"
+#endif
#elif defined(__GNUC__)
#pragma GCC diagnostic ignored "-Wunused-macros"
#pragma GCC diagnostic ignored "-Wunused-function"
#endif
+#if !defined(__has_builtin)
+#define __has_builtin(b) 0
+#endif
+
+#if defined(__GNUC__) || defined(__clang__)
+
+#if __has_builtin(__builtin_memcpy_inline)
+#define _rpmalloc_memcpy_const(x, y, s) __builtin_memcpy_inline(x, y, s)
+#else
+#define _rpmalloc_memcpy_const(x, y, s) \
+ do { \
+ _Static_assert(__builtin_choose_expr(__builtin_constant_p(s), 1, 0), \
+ "len must be a constant integer"); \
+ memcpy(x, y, s); \
+ } while (0)
+#endif
+
+#if __has_builtin(__builtin_memset_inline)
+#define _rpmalloc_memset_const(x, y, s) __builtin_memset_inline(x, y, s)
+#else
+#define _rpmalloc_memset_const(x, y, s) \
+ do { \
+ _Static_assert(__builtin_choose_expr(__builtin_constant_p(s), 1, 0), \
+ "len must be a constant integer"); \
+ memset(x, y, s); \
+ } while (0)
+#endif
+#else
+#define _rpmalloc_memcpy_const(x, y, s) memcpy(x, y, s)
+#define _rpmalloc_memset_const(x, y, s) memset(x, y, s)
+#endif
+
+#if __has_builtin(__builtin_assume)
+#define rpmalloc_assume(cond) __builtin_assume(cond)
+#elif defined(__GNUC__)
+#define rpmalloc_assume(cond) \
+ do { \
+ if (!__builtin_expect(cond, 0)) \
+ __builtin_unreachable(); \
+ } while (0)
+#elif defined(_MSC_VER)
+#define rpmalloc_assume(cond) __assume(cond)
+#else
+#define rpmalloc_assume(cond) 0
+#endif
+
#ifndef HEAP_ARRAY_SIZE
//! Size of heap hashmap
#define HEAP_ARRAY_SIZE 47
@@ -444,17 +494,17 @@ _Static_assert((SPAN_HEADER_SIZE & (SPAN_HEADER_SIZE - 1)) == 0,
#if ENABLE_VALIDATE_ARGS
//! Maximum allocation size to avoid integer overflow
#undef MAX_ALLOC_SIZE
-#define MAX_ALLOC_SIZE (((size_t)-1) - _memory_span_size)
+#define MAX_ALLOC_SIZE (((size_t) - 1) - _memory_span_size)
#endif
#define pointer_offset(ptr, ofs) (void *)((char *)(ptr) + (ptrdiff_t)(ofs))
#define pointer_diff(first, second) \
(ptrdiff_t)((const char *)(first) - (const char *)(second))
-#define INVALID_POINTER ((void *)((uintptr_t)-1))
+#define INVALID_POINTER ((void *)((uintptr_t) - 1))
#define SIZE_CLASS_LARGE SIZE_CLASS_COUNT
-#define SIZE_CLASS_HUGE ((uint32_t)-1)
+#define SIZE_CLASS_HUGE ((uint32_t) - 1)
////////////
///
@@ -866,11 +916,11 @@ static inline uintptr_t get_thread_id(void) {
__asm__ volatile("mrs %0, tpidr_el0" : "=r"(tid));
#endif
#else
- tid = (uintptr_t)((void *)get_thread_heap_raw());
+#error This platform needs implementation of get_thread_id()
#endif
return tid;
#else
- return (uintptr_t)((void *)get_thread_heap_raw());
+#error This platform needs implementation of get_thread_id()
#endif
}
@@ -895,7 +945,11 @@ void rpmalloc_set_main_thread(void) {
static void _rpmalloc_spin(void) {
#if defined(_MSC_VER)
+#if defined(_M_ARM64)
+ __yield();
+#else
_mm_pause();
+#endif
#elif defined(__x86_64__) || defined(__i386__)
__asm__ volatile("pause" ::: "memory");
#elif defined(__aarch64__) || (defined(__arm__) && __ARM_ARCH >= 7)
@@ -1698,7 +1752,8 @@ static size_t _rpmalloc_global_cache_extract_spans(span_t **span,
#if ENABLE_ASSERTS
for (size_t ispan = 0; ispan < extract_count; ++ispan) {
- assert(span[ispan]->span_count == span_count);
+ rpmalloc_assert(span[ispan]->span_count == span_count,
+ "Global cache span count mismatch");
}
#endif
@@ -2028,15 +2083,15 @@ _rpmalloc_heap_extract_new_span(heap_t *heap,
_rpmalloc_inc_span_statistics(heap, span_count, class_idx);
return span;
}
- span = _rpmalloc_heap_reserved_extract(heap, span_count);
+ span = _rpmalloc_heap_global_cache_extract(heap, span_count);
if (EXPECTED(span != 0)) {
- _rpmalloc_stat_inc(&heap->size_class_use[class_idx].spans_from_reserved);
+ _rpmalloc_stat_inc(&heap->size_class_use[class_idx].spans_from_cache);
_rpmalloc_inc_span_statistics(heap, span_count, class_idx);
return span;
}
- span = _rpmalloc_heap_global_cache_extract(heap, span_count);
+ span = _rpmalloc_heap_reserved_extract(heap, span_count);
if (EXPECTED(span != 0)) {
- _rpmalloc_stat_inc(&heap->size_class_use[class_idx].spans_from_cache);
+ _rpmalloc_stat_inc(&heap->size_class_use[class_idx].spans_from_reserved);
_rpmalloc_inc_span_statistics(heap, span_count, class_idx);
return span;
}
@@ -2050,7 +2105,7 @@ _rpmalloc_heap_extract_new_span(heap_t *heap,
}
static void _rpmalloc_heap_initialize(heap_t *heap) {
- memset(heap, 0, sizeof(heap_t));
+ _rpmalloc_memset_const(heap, 0, sizeof(heap_t));
// Get a new heap ID
heap->id = 1 + atomic_incr32(&_memory_heap_id);
@@ -2179,7 +2234,8 @@ static heap_t *_rpmalloc_heap_allocate(int first_class) {
if (!heap)
heap = _rpmalloc_heap_allocate_new();
atomic_store32_release(&_memory_global_lock, 0);
- _rpmalloc_heap_cache_adopt_deferred(heap, 0);
+ if (heap)
+ _rpmalloc_heap_cache_adopt_deferred(heap, 0);
return heap;
}
@@ -2321,6 +2377,7 @@ static void *free_list_pop(void **list) {
static void *_rpmalloc_allocate_from_heap_fallback(
heap_t *heap, heap_size_class_t *heap_size_class, uint32_t class_idx) {
span_t *span = heap_size_class->partial_span;
+ rpmalloc_assume(heap != 0);
if (EXPECTED(span != 0)) {
rpmalloc_assert(span->block_count ==
_memory_size_class[span->size_class].block_count,
@@ -2498,7 +2555,8 @@ static void *_rpmalloc_aligned_allocate(heap_t *heap, size_t alignment,
}
#endif
- if ((alignment <= SPAN_HEADER_SIZE) && (size < _memory_medium_size_limit)) {
+ if ((alignment <= SPAN_HEADER_SIZE) &&
+ ((size + SPAN_HEADER_SIZE) < _memory_medium_size_limit)) {
// If alignment is less or equal to span header size (which is power of
// two), and size aligned to span header size multiples is less than size +
// alignment, then use natural alignment of blocks to provide alignment
@@ -2977,8 +3035,9 @@ static void _rpmalloc_adjust_size_class(size_t iclass) {
// A class can be merged if number of pages and number of blocks are equal
if (_memory_size_class[prevclass].block_count ==
_memory_size_class[iclass].block_count)
- memcpy(_memory_size_class + prevclass, _memory_size_class + iclass,
- sizeof(_memory_size_class[iclass]));
+ _rpmalloc_memcpy_const(_memory_size_class + prevclass,
+ _memory_size_class + iclass,
+ sizeof(_memory_size_class[iclass]));
else
break;
}
@@ -3004,7 +3063,7 @@ int rpmalloc_initialize_config(const rpmalloc_config_t *config) {
if (config)
memcpy(&_memory_config, config, sizeof(rpmalloc_config_t));
else
- memset(&_memory_config, 0, sizeof(rpmalloc_config_t));
+ _rpmalloc_memset_const(&_memory_config, 0, sizeof(rpmalloc_config_t));
if (!_memory_config.memory_map || !_memory_config.memory_unmap) {
_memory_config.memory_map = _rpmalloc_mmap_os;
@@ -3055,8 +3114,24 @@ int rpmalloc_initialize_config(const rpmalloc_config_t *config) {
if (sysctlbyname("vm.pmap.pg_ps_enabled", &rc, &sz, NULL, 0) == 0 &&
rc == 1) {
+ static size_t defsize = 2 * 1024 * 1024;
+ int nsize = 0;
+ size_t sizes[4] = {0};
_memory_huge_pages = 1;
- _memory_page_size = 2 * 1024 * 1024;
+ _memory_page_size = defsize;
+ if ((nsize = getpagesizes(sizes, 4)) >= 2) {
+ nsize--;
+ for (size_t csize = sizes[nsize]; nsize >= 0 && csize;
+ --nsize, csize = sizes[nsize]) {
+ //! Unlikely, but as a precaution..
+ rpmalloc_assert(!(csize & (csize - 1)) && !(csize % 1024),
+ "Invalid page size");
+ if (defsize < csize) {
+ _memory_page_size = csize;
+ break;
+ }
+ }
+ }
_memory_map_granularity = _memory_page_size;
}
#elif defined(__APPLE__) || defined(__NetBSD__)
@@ -3181,8 +3256,11 @@ int rpmalloc_initialize_config(const rpmalloc_config_t *config) {
_memory_medium_size_limit = MEDIUM_SIZE_LIMIT;
for (iclass = 0; iclass < MEDIUM_CLASS_COUNT; ++iclass) {
size_t size = SMALL_SIZE_LIMIT + ((iclass + 1) * MEDIUM_GRANULARITY);
- if (size > _memory_medium_size_limit)
+ if (size > _memory_medium_size_limit) {
+ _memory_medium_size_limit =
+ SMALL_SIZE_LIMIT + (iclass * MEDIUM_GRANULARITY);
break;
+ }
_memory_size_class[SMALL_CLASS_COUNT + iclass].block_size = (uint32_t)size;
_rpmalloc_adjust_size_class(SMALL_CLASS_COUNT + iclass);
}
@@ -3205,6 +3283,8 @@ int rpmalloc_initialize_config(const rpmalloc_config_t *config) {
memset(_memory_heaps, 0, sizeof(_memory_heaps));
atomic_store32_release(&_memory_global_lock, 0);
+ rpmalloc_linker_reference();
+
// Initialize this thread
rpmalloc_thread_initialize();
return 0;
@@ -3422,7 +3502,7 @@ void rpmalloc_thread_statistics(rpmalloc_thread_statistics_t *stats) {
if (span->free_list_limit < block_count)
block_count = span->free_list_limit;
free_count += (block_count - span->used_count);
- stats->sizecache = free_count * size_class->block_size;
+ stats->sizecache += free_count * size_class->block_size;
span = span->next;
}
}
@@ -3434,14 +3514,14 @@ void rpmalloc_thread_statistics(rpmalloc_thread_statistics_t *stats) {
span_cache = &heap->span_cache;
else
span_cache = (span_cache_t *)(heap->span_large_cache + (iclass - 1));
- stats->spancache = span_cache->count * (iclass + 1) * _memory_span_size;
+ stats->spancache += span_cache->count * (iclass + 1) * _memory_span_size;
}
#endif
span_t *deferred = (span_t *)atomic_load_ptr(&heap->span_free_deferred);
while (deferred) {
if (deferred->size_class != SIZE_CLASS_HUGE)
- stats->spancache = (size_t)deferred->span_count * _memory_span_size;
+ stats->spancache += (size_t)deferred->span_count * _memory_span_size;
deferred = (span_t *)deferred->free_list;
}
@@ -3504,9 +3584,21 @@ void rpmalloc_global_statistics(rpmalloc_global_statistics_t *stats) {
stats->huge_alloc_peak = (size_t)_huge_pages_peak * _memory_page_size;
#endif
#if ENABLE_GLOBAL_CACHE
- for (size_t iclass = 0; iclass < LARGE_CLASS_COUNT; ++iclass)
- stats->cached +=
- _memory_span_cache[iclass].count * (iclass + 1) * _memory_span_size;
+ for (size_t iclass = 0; iclass < LARGE_CLASS_COUNT; ++iclass) {
+ global_cache_t *cache = &_memory_span_cache[iclass];
+ while (!atomic_cas32_acquire(&cache->lock, 1, 0))
+ _rpmalloc_spin();
+ uint32_t count = cache->count;
+#if ENABLE_UNLIMITED_CACHE
+ span_t *current_span = cache->overflow;
+ while (current_span) {
+ ++count;
+ current_span = current_span->next;
+ }
+#endif
+ atomic_store32_release(&cache->lock, 0);
+ stats->cached += count * (iclass + 1) * _memory_span_size;
+ }
#endif
}
@@ -3639,6 +3731,7 @@ void rpmalloc_dump_statistics(void *file) {
fprintf(file, "%14zu %11zu\n", huge_current / (size_t)(1024 * 1024),
huge_peak / (size_t)(1024 * 1024));
+#if ENABLE_GLOBAL_CACHE
fprintf(file, "GlobalCacheMiB\n");
for (size_t iclass = 0; iclass < LARGE_CLASS_COUNT; ++iclass) {
global_cache_t *cache = _memory_span_cache + iclass;
@@ -3658,6 +3751,7 @@ void rpmalloc_dump_statistics(void *file) {
global_overflow_cache / (size_t)(1024 * 1024),
cache->insert_count, cache->extract_count);
}
+#endif
size_t mapped = (size_t)atomic_load32(&_mapped_pages) * _memory_page_size;
size_t mapped_os =
@@ -3698,6 +3792,7 @@ extern inline rpmalloc_heap_t *rpmalloc_heap_acquire(void) {
// released when heap is cleared with rpmalloc_heap_free_all(). Also heaps
// guaranteed to be pristine from the dedicated orphan list can be used.
heap_t *heap = _rpmalloc_heap_allocate(1);
+ rpmalloc_assume(heap != NULL);
heap->owner_thread = 0;
_rpmalloc_stat_inc(&_memory_active_heaps);
return heap;
@@ -3813,6 +3908,11 @@ extern inline void rpmalloc_heap_free_all(rpmalloc_heap_t *heap) {
_rpmalloc_heap_cache_insert(heap, span);
span = next_span;
}
+
+ span = heap->size_class[iclass].cache;
+ if (span)
+ _rpmalloc_heap_cache_insert(heap, span);
+ heap->size_class[iclass].cache = 0;
}
memset(heap->size_class, 0, sizeof(heap->size_class));
memset(heap->full_span, 0, sizeof(heap->full_span));
@@ -3873,6 +3973,15 @@ extern inline void rpmalloc_heap_thread_set_current(rpmalloc_heap_t *heap) {
}
}
+extern inline rpmalloc_heap_t *rpmalloc_get_heap_for_ptr(void *ptr) {
+ // Grab the span, and then the heap from the span
+ span_t *span = (span_t *)((uintptr_t)ptr & _memory_span_mask);
+ if (span) {
+ return span->heap;
+ }
+ return 0;
+}
+
#endif
#if ENABLE_PRELOAD || ENABLE_OVERRIDE
@@ -3880,3 +3989,5 @@ extern inline void rpmalloc_heap_thread_set_current(rpmalloc_heap_t *heap) {
#include "malloc.c"
#endif
+
+void rpmalloc_linker_reference(void) { (void)sizeof(_rpmalloc_initialized); }
diff --git a/llvm/lib/Support/rpmalloc/rpmalloc.h b/llvm/lib/Support/rpmalloc/rpmalloc.h
index a57bc4a6a2ece..b2c47f66fda3c 100644
--- a/llvm/lib/Support/rpmalloc/rpmalloc.h
+++ b/llvm/lib/Support/rpmalloc/rpmalloc.h
@@ -1,4 +1,5 @@
-//===----------------------- rpmalloc.h -------------------*- C -*-===========//
+//===------------------------ rpmalloc.h -----------------*- C
+//-*-=============//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -329,6 +330,9 @@ RPMALLOC_EXPORT int rpposix_memalign(void **memptr, size_t alignment,
//! end of block)
RPMALLOC_EXPORT size_t rpmalloc_usable_size(void *ptr);
+//! Dummy empty function for forcing linker symbol inclusion
+RPMALLOC_EXPORT void rpmalloc_linker_reference(void);
+
#if RPMALLOC_FIRST_CLASS_HEAPS
//! Heap type
@@ -415,6 +419,9 @@ RPMALLOC_EXPORT void rpmalloc_heap_free_all(rpmalloc_heap_t *heap);
// other threads.
RPMALLOC_EXPORT void rpmalloc_heap_thread_set_current(rpmalloc_heap_t *heap);
+//! Returns which heap the given pointer is allocated on
+RPMALLOC_EXPORT rpmalloc_heap_t *rpmalloc_get_heap_for_ptr(void *ptr);
+
#endif
#ifdef __cplusplus
diff --git a/llvm/lib/Support/rpmalloc/rpnew.h b/llvm/lib/Support/rpmalloc/rpnew.h
index 51786ade5a3ea..d8303c6f95652 100644
--- a/llvm/lib/Support/rpmalloc/rpnew.h
+++ b/llvm/lib/Support/rpmalloc/rpnew.h
@@ -1,18 +1,24 @@
-//===---------------------------- rpnew.h ------------------*- C
-//-*-===========//
+//===-------------------------- rpnew.h -----------------*- C -*-=============//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
+//
+// This library provides a cross-platform lock free thread caching malloc
+// implementation in C11.
+//
+//===----------------------------------------------------------------------===//
#ifdef __cplusplus
#include <new>
#include <rpmalloc.h>
-#ifdef _WIN32
+#ifndef __CRTDECL
+#define __CRTDECL
+#endif
extern void __CRTDECL operator delete(void *p) noexcept { rpfree(p); }
@@ -82,28 +88,26 @@ extern void __CRTDECL operator delete[](void *p, std::size_t size,
extern void *__CRTDECL operator new(std::size_t size,
std::align_val_t align) noexcept(false) {
- return rpaligned_alloc((size_t)align, size);
+ return rpaligned_alloc(static_cast<size_t>(align), size);
}
extern void *__CRTDECL operator new[](std::size_t size,
std::align_val_t align) noexcept(false) {
- return rpaligned_alloc((size_t)align, size);
+ return rpaligned_alloc(static_cast<size_t>(align), size);
}
extern void *__CRTDECL operator new(std::size_t size, std::align_val_t align,
const std::nothrow_t &tag) noexcept {
(void)sizeof(tag);
- return rpaligned_alloc((size_t)align, size);
+ return rpaligned_alloc(static_cast<size_t>(align), size);
}
extern void *__CRTDECL operator new[](std::size_t size, std::align_val_t align,
const std::nothrow_t &tag) noexcept {
(void)sizeof(tag);
- return rpaligned_alloc((size_t)align, size);
+ return rpaligned_alloc(static_cast<size_t>(align), size);
}
#endif
#endif
-
-#endif
>From 38926692f050ebd55bb5198b8df71e0705ed140d Mon Sep 17 00:00:00 2001
From: Alexandre Ganea <aganea at havenstudios.com>
Date: Mon, 13 May 2024 08:42:49 -0400
Subject: [PATCH 4/8] Add rpmalloc version number 1.4.5
---
llvm/lib/Support/rpmalloc/README.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/lib/Support/rpmalloc/README.md b/llvm/lib/Support/rpmalloc/README.md
index 6173876a0f632..916bca0118d86 100644
--- a/llvm/lib/Support/rpmalloc/README.md
+++ b/llvm/lib/Support/rpmalloc/README.md
@@ -1,5 +1,6 @@
# rpmalloc - General Purpose Memory Allocator
This library provides a cross platform lock free thread caching 16-byte aligned memory allocator implemented in C.
+This is a fork of rpmalloc 1.4.5.
Platforms currently supported:
>From 42dfd62538cf62b96832b310010e046b75137215 Mon Sep 17 00:00:00 2001
From: Alexandre Ganea <aganea at havenstudios.com>
Date: Mon, 13 May 2024 08:54:52 -0400
Subject: [PATCH 5/8] More clang-format fixes.
---
llvm/lib/Support/rpmalloc/malloc.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Support/rpmalloc/malloc.c b/llvm/lib/Support/rpmalloc/malloc.c
index 498aa2782586c..3fcfe848250c6 100644
--- a/llvm/lib/Support/rpmalloc/malloc.c
+++ b/llvm/lib/Support/rpmalloc/malloc.c
@@ -551,11 +551,11 @@ extern inline void *RPMALLOC_CDECL pvalloc(size_t size) {
#if defined(BUILD_DYNAMIC_LINK) && BUILD_DYNAMIC_LINK
extern __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE instance,
- DWORD reason,
- LPVOID reserved);
+ DWORD reason, LPVOID reserved);
-extern __declspec(dllexport) BOOL WINAPI
-DllMain(HINSTANCE instance, DWORD reason, LPVOID reserved) {
+extern __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE instance,
+ DWORD reason,
+ LPVOID reserved) {
(void)sizeof(reserved);
(void)sizeof(instance);
if (reason == DLL_PROCESS_ATTACH)
>From 43396f3b73f5022c3006756099b88de48fc1ff59 Mon Sep 17 00:00:00 2001
From: Alexandre Ganea <aganea at havenstudios.com>
Date: Mon, 13 May 2024 10:33:04 -0400
Subject: [PATCH 6/8] Clarify documentation a bit
---
llvm/docs/CMake.rst | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/llvm/docs/CMake.rst b/llvm/docs/CMake.rst
index 0b65e06d1d7b3..83ceeebc09eaf 100644
--- a/llvm/docs/CMake.rst
+++ b/llvm/docs/CMake.rst
@@ -712,16 +712,17 @@ enabled sub-projects. Nearly all of these variable names begin with
This flag needs to be used along with the static CRT, ie. if building the
Release target, add -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded.
- Note that rpmalloc is also supported natively in-tree, for example:
-
- .. code-block:: console
-
- $ D:\llvm-project> cmake ... -DLLVM_ENABLE_RPMALLOC=ON
+ Note that rpmalloc is also supported natively in-tree, see option below.
**LLVM_ENABLE_RPMALLOC**:BOOL
Similar to LLVM_INTEGRATED_CRT_ALLOC, embeds the in-tree rpmalloc into the
host toolchain as a C runtime allocator. Requires linking with the static CRT,
- if building the Release target, with: -DLLVM_USE_CRT_RELEASE=MT.
+ for example when building the Release target:
+
+ .. code-block:: console
+
+ $ D:\llvm-project> cmake ... -DLLVM_ENABLE_RPMALLOC=ON ^
+ -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded
**LLVM_LINK_LLVM_DYLIB**:BOOL
If enabled, tools will be linked with the libLLVM shared library. Defaults
>From 9ff1baf78174138c0e303867e572947ea87a49f9 Mon Sep 17 00:00:00 2001
From: Alexandre Ganea <aganea at havenstudios.com>
Date: Sat, 18 May 2024 10:38:06 -0400
Subject: [PATCH 7/8] Add a warning for multi-config cmake generators
---
llvm/CMakeLists.txt | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index 4f20f8dbb1849..68d9d6fa34b70 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -741,8 +741,19 @@ if(LLVM_ENABLE_RPMALLOC)
if(LLVM_USE_SANITIZER)
message(FATAL_ERROR "LLVM_ENABLE_RPMALLOC cannot be used along with LLVM_USE_SANITIZER!")
endif()
- if(WIN32 AND CMAKE_BUILD_TYPE AND uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG")
- message(FATAL_ERROR "The Debug target isn't supported along with LLVM_ENABLE_RPMALLOC!")
+ if(WIN32)
+ if(CMAKE_CONFIGURATION_TYPES)
+ foreach(BUILD_MODE ${CMAKE_CONFIGURATION_TYPES})
+ string(TOUPPER "${BUILD_MODE}" uppercase_BUILD_MODE)
+ if(uppercase_BUILD_MODE STREQUAL "DEBUG")
+ message(WARNING "The Debug target isn't supported along with LLVM_ENABLE_RPMALLOC!")
+ endif()
+ endforeach()
+ else()
+ if(CMAKE_BUILD_TYPE AND uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+ message(FATAL_ERROR "The Debug target isn't supported along with LLVM_ENABLE_RPMALLOC!")
+ endif()
+ endif()
endif()
# Override the C runtime allocator with the in-tree rpmalloc
>From ac63d809952e0ebb287ef65cb00e29f3e2403a1f Mon Sep 17 00:00:00 2001
From: Alexandre Ganea <aganea at havenstudios.com>
Date: Sat, 18 May 2024 10:53:30 -0400
Subject: [PATCH 8/8] Slightly change wording in the documentation.
---
llvm/docs/CMake.rst | 12 ++++--------
llvm/docs/ReleaseNotes.rst | 2 +-
2 files changed, 5 insertions(+), 9 deletions(-)
diff --git a/llvm/docs/CMake.rst b/llvm/docs/CMake.rst
index 83ceeebc09eaf..2a80813999ea1 100644
--- a/llvm/docs/CMake.rst
+++ b/llvm/docs/CMake.rst
@@ -710,19 +710,15 @@ enabled sub-projects. Nearly all of these variable names begin with
$ D:\git> git clone https://github.com/mjansson/rpmalloc
$ D:\llvm-project> cmake ... -DLLVM_INTEGRATED_CRT_ALLOC=D:\git\rpmalloc
- This flag needs to be used along with the static CRT, ie. if building the
+ This option needs to be used along with the static CRT, ie. if building the
Release target, add -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded.
Note that rpmalloc is also supported natively in-tree, see option below.
**LLVM_ENABLE_RPMALLOC**:BOOL
Similar to LLVM_INTEGRATED_CRT_ALLOC, embeds the in-tree rpmalloc into the
- host toolchain as a C runtime allocator. Requires linking with the static CRT,
- for example when building the Release target:
-
- .. code-block:: console
-
- $ D:\llvm-project> cmake ... -DLLVM_ENABLE_RPMALLOC=ON ^
- -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded
+ host toolchain as a C runtime allocator. The version currently used is
+ rpmalloc 1.4.5. This option also implies linking with the static CRT, there's
+ no need to provide CMAKE_MSVC_RUNTIME_LIBRARY.
**LLVM_LINK_LLVM_DYLIB**:BOOL
If enabled, tools will be linked with the libLLVM shared library. Defaults
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index 35910e20b3fff..660c107751bf3 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -63,7 +63,7 @@ Changes to building LLVM
------------------------
* LLVM now supports rpmalloc in-tree, as a replacement C allocator for hosted
- toolchains.
+ toolchains. Currently using version 1.4.5.
Changes to TableGen
-------------------
More information about the llvm-commits
mailing list