[libc-commits] [libc] [libc][malloc] Reduce block overhead by 4 bytes plus alignment effects (PR #99945)
Daniel Thornburgh via libc-commits
libc-commits at lists.llvm.org
Tue Jul 23 12:06:32 PDT 2024
https://github.com/mysterymath updated https://github.com/llvm/llvm-project/pull/99945
>From 285cc08f816cf11a492613ad67bce74fe24fbc09 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Tue, 9 Jul 2024 14:53:28 -0700
Subject: [PATCH 1/2] [libc][malloc] Reduce block overhead by 8 bytes (on
32-bit)
The unused padding and alignment fields were removed. The used and last
bits were stashed into the lower two bits of the next chunk offset,
which were recast as being actual offsets rather than being in units of
the alignment. (This is a very typical trick for Knuth boundary tags.)
To ensure that the lowest two bits are not significant, the minimum
alignment was bumped to 4. This shouldn't affect anything in practice,
since alignof(max_align_t) is overwhelmingly likely to be 8.
---
libc/src/__support/block.h | 126 +++++++++++++++----------------------
1 file changed, 50 insertions(+), 76 deletions(-)
diff --git a/libc/src/__support/block.h b/libc/src/__support/block.h
index 242602ad2f856..a4d7cc6628502 100644
--- a/libc/src/__support/block.h
+++ b/libc/src/__support/block.h
@@ -65,62 +65,53 @@ using cpp::optional;
/// Memory region with links to adjacent blocks.
///
-/// The blocks do not encode their size directly. Instead, they encode offsets
-/// to the next and previous blocks using the type given by the `OffsetType`
-/// template parameter. The encoded offsets are simply the offsets divded by the
-/// minimum block alignment, `ALIGNMENT`.
+/// The blocks store their offsets to the previous and next blocks. The latter
+/// is also the block's size.
///
/// The `ALIGNMENT` constant provided by the derived block is typically the
-/// minimum value of `alignof(OffsetType)`. Since the addressable range of a
-/// block is given by `std::numeric_limits<OffsetType>::max() *
-/// ALIGNMENT`, it may be advantageous to set a higher alignment if it allows
-/// using a smaller offset type, even if this wastes some bytes in order to
-/// align block headers.
-///
-/// Blocks will always be aligned to a `ALIGNMENT` boundary. Block sizes will
-/// always be rounded up to a multiple of `ALIGNMENT`.
+/// minimum value of `alignof(OffsetType)`. Blocks will always be aligned to a
+/// `ALIGNMENT` boundary. Block sizes will always be rounded up to a multiple of
+/// `ALIGNMENT`.
///
/// As an example, the diagram below represents two contiguous
/// `Block<uint32_t, 8>`s. The indices indicate byte offsets:
///
/// @code{.unparsed}
/// Block 1:
-/// +---------------------+------+--------------+
-/// | Header | Info | Usable space |
-/// +----------+----------+------+--------------+
-/// | prev | next | | |
-/// | 0......3 | 4......7 | 8..9 | 10.......280 |
-/// | 00000000 | 00000046 | 8008 | <app data> |
-/// +----------+----------+------+--------------+
+/// +---------------------+--------------+
+/// | Header | Usable space |
+/// +----------+----------+--------------+
+/// | prev | next | |
+/// | 0......3 | 4......7 | 8........227 |
+/// | 00000000 | 00000230 | <app data> |
+/// +----------+----------+--------------+
/// Block 2:
-/// +---------------------+------+--------------+
-/// | Header | Info | Usable space |
-/// +----------+----------+------+--------------+
-/// | prev | next | | |
-/// | 0......3 | 4......7 | 8..9 | 10......1056 |
-/// | 00000046 | 00000106 | 2008 | f7f7....f7f7 |
-/// +----------+----------+------+--------------+
+/// +---------------------+--------------+
+/// | Header | Usable space |
+/// +----------+----------+--------------+
+/// | prev | next | |
+/// | 0......3 | 4......7 | 8........827 |
+/// | 00000230 | 00000830 | f7f7....f7f7 |
+/// +----------+----------+--------------+
/// @endcode
///
-/// The overall size of the block (e.g. 280 bytes) is given by its next offset
-/// multiplied by the alignment (e.g. 0x106 * 4). Also, the next offset of a
-/// block matches the previous offset of its next block. The first block in a
-/// list is denoted by having a previous offset of `0`.
+/// The next offset of a block matches the previous offset of its next block.
+/// The first block in a list is denoted by having a previous offset of `0`.
///
/// @tparam OffsetType Unsigned integral type used to encode offsets. Larger
/// types can address more memory, but consume greater
/// overhead.
/// @tparam kAlign Sets the overall alignment for blocks. Minimum is
-/// `alignof(OffsetType)` (the default). Larger values can
-/// address more memory, but consume greater overhead.
+/// `alignof(OffsetType)` (the default). Larger values
+/// cause greater overhead.
template <typename OffsetType = uintptr_t, size_t kAlign = alignof(OffsetType)>
class Block {
public:
using offset_type = OffsetType;
static_assert(cpp::is_unsigned_v<offset_type>,
"offset type must be unsigned");
-
- static constexpr size_t ALIGNMENT = cpp::max(kAlign, alignof(offset_type));
+ static constexpr size_t ALIGNMENT =
+ cpp::max(cpp::max(kAlign, alignof(offset_type)), size_t{4});
static constexpr size_t BLOCK_OVERHEAD = align_up(sizeof(Block), ALIGNMENT);
// No copy or move.
@@ -147,14 +138,11 @@ class Block {
}
/// @returns The total size of the block in bytes, including the header.
- size_t outer_size() const { return next_ * ALIGNMENT; }
+ size_t outer_size() const { return next_ & ~3u; }
/// @returns The number of usable bytes inside the block.
size_t inner_size() const { return outer_size() - BLOCK_OVERHEAD; }
- /// @returns The number of bytes requested using AllocFirst or AllocLast.
- size_t requested_size() const { return inner_size() - padding_; }
-
/// @returns A pointer to the usable space inside this block.
cpp::byte *usable_space() {
return reinterpret_cast<cpp::byte *>(this) + BLOCK_OVERHEAD;
@@ -224,13 +212,10 @@ class Block {
return block == nullptr ? nullptr : block->prev();
}
- /// Returns the current alignment of a block.
- size_t alignment() const { return used() ? info_.alignment : 1; }
-
/// Indicates whether the block is in use.
///
/// @returns `true` if the block is in use or `false` if not.
- bool used() const { return info_.used; }
+ bool used() const { return next_ & 1; }
/// Indicates whether this block is the last block or not (i.e. whether
/// `next()` points to a valid block or not). This is needed because
@@ -238,19 +223,19 @@ class Block {
/// block there or not.
///
/// @returns `true` is this is the last block or `false` if not.
- bool last() const { return info_.last; }
+ bool last() const { return next_ & 2; }
/// Marks this block as in use.
- void mark_used() { info_.used = 1; }
+ void mark_used() { next_ |= 1; }
/// Marks this block as free.
- void mark_free() { info_.used = 0; }
+ void mark_free() { next_ &= ~1u; }
/// Marks this block as the last one in the chain.
- constexpr void mark_last() { info_.last = 1; }
+ constexpr void mark_last() { next_ |= 2; }
/// Clears the last bit from this block.
- void clear_last() { info_.last = 1; }
+ void clear_last() { next_ &= ~2u; }
/// @brief Checks if a block is valid.
///
@@ -338,32 +323,26 @@ class Block {
/// ensure the split will succeed.
static Block *split_impl(Block *&block, size_t new_inner_size);
- /// Offset (in increments of the minimum alignment) from this block to the
- /// previous block. 0 if this is the first block.
+ /// Offset from this block to the previous block. 0 if this is the first
+ /// block.
offset_type prev_ = 0;
- /// Offset (in increments of the minimum alignment) from this block to the
- /// next block. Valid even if this is the last block, since it equals the
- /// size of the block.
+ /// Offset from this block to the next block. Valid even if this is the last
+ /// block, since it equals the size of the block.
offset_type next_ = 0;
- /// Information about the current state of the block:
+ /// Information about the current state of the block is stored in the two low
+ /// order bits of the next_ value. These are guaranteed free by a minimum
+ /// alignment (and thus, alignment of the size) of 4. The lowest bit is the
+ /// `used` flag, and the other bit is the `last` flag.
+ ///
/// * If the `used` flag is set, the block's usable memory has been allocated
/// and is being used.
/// * If the `last` flag is set, the block does not have a next block.
/// * If the `used` flag is set, the alignment represents the requested value
/// when the memory was allocated, which may be less strict than the actual
/// alignment.
- struct {
- uint16_t used : 1;
- uint16_t last : 1;
- uint16_t alignment : 14;
- } info_;
-
- /// Number of bytes allocated beyond what was requested. This will be at most
- /// the minimum alignment, i.e. `alignof(offset_type).`
- uint16_t padding_ = 0;
-} __attribute__((packed, aligned(kAlign)));
+} __attribute__((packed, aligned(cpp::max(kAlign, size_t{4}))));
// Public template method implementations.
@@ -394,7 +373,7 @@ Block<OffsetType, kAlign>::init(ByteSpan region) {
if (region.size() < BLOCK_OVERHEAD)
return {};
- if (cpp::numeric_limits<OffsetType>::max() < region.size() / ALIGNMENT)
+ if (cpp::numeric_limits<OffsetType>::max() < region.size())
return {};
Block *block = as_block(0, region);
@@ -501,7 +480,7 @@ Block<OffsetType, kAlign>::split(Block *&block, size_t new_inner_size) {
template <typename OffsetType, size_t kAlign>
Block<OffsetType, kAlign> *
Block<OffsetType, kAlign>::split_impl(Block *&block, size_t new_inner_size) {
- size_t prev_outer_size = block->prev_ * ALIGNMENT;
+ size_t prev_outer_size = block->prev_;
size_t outer_size1 = new_inner_size + BLOCK_OVERHEAD;
bool is_last = block->last();
ByteSpan bytes = as_bytes(cpp::move(block));
@@ -529,7 +508,7 @@ bool Block<OffsetType, kAlign>::merge_next(Block *&block) {
if (block->used() || next->used())
return false;
- size_t prev_outer_size = block->prev_ * ALIGNMENT;
+ size_t prev_outer_size = block->prev_;
bool is_last = next->last();
ByteSpan prev_bytes = as_bytes(cpp::move(block));
ByteSpan next_bytes = as_bytes(cpp::move(next));
@@ -554,9 +533,7 @@ Block<OffsetType, kAlign> *Block<OffsetType, kAlign>::next() const {
template <typename OffsetType, size_t kAlign>
Block<OffsetType, kAlign> *Block<OffsetType, kAlign>::prev() const {
- uintptr_t addr =
- (prev_ == 0) ? 0
- : reinterpret_cast<uintptr_t>(this) - (prev_ * ALIGNMENT);
+ uintptr_t addr = (prev_ == 0) ? 0 : reinterpret_cast<uintptr_t>(this) - prev_;
return reinterpret_cast<Block *>(addr);
}
@@ -564,13 +541,10 @@ Block<OffsetType, kAlign> *Block<OffsetType, kAlign>::prev() const {
template <typename OffsetType, size_t kAlign>
constexpr Block<OffsetType, kAlign>::Block(size_t prev_outer_size,
- size_t outer_size)
- : info_{} {
- prev_ = prev_outer_size / ALIGNMENT;
- next_ = outer_size / ALIGNMENT;
- info_.used = 0;
- info_.last = 0;
- info_.alignment = ALIGNMENT;
+ size_t outer_size) {
+ prev_ = prev_outer_size;
+ LIBC_ASSERT(outer_size % ALIGNMENT == 0 && "block sizes must be aligned");
+ next_ = outer_size;
}
template <typename OffsetType, size_t kAlign>
>From bb76e9b4acbea1830f0d01f1e72eba0668c08b9b Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Tue, 23 Jul 2024 12:04:38 -0700
Subject: [PATCH 2/2] Use named masks for bit manipulation
---
libc/src/__support/block.h | 19 ++++++++++++-------
1 file changed, 12 insertions(+), 7 deletions(-)
diff --git a/libc/src/__support/block.h b/libc/src/__support/block.h
index a4d7cc6628502..af3ce181f1c99 100644
--- a/libc/src/__support/block.h
+++ b/libc/src/__support/block.h
@@ -106,6 +106,11 @@ using cpp::optional;
/// cause greater overhead.
template <typename OffsetType = uintptr_t, size_t kAlign = alignof(OffsetType)>
class Block {
+ // Masks for the contents of the next_ field.
+ static constexpr size_t USED_MASK = 1 << 0;
+ static constexpr size_t LAST_MASK = 1 << 1;
+ static constexpr size_t SIZE_MASK = ~(USED_MASK | LAST_MASK);
+
public:
using offset_type = OffsetType;
static_assert(cpp::is_unsigned_v<offset_type>,
@@ -138,7 +143,7 @@ class Block {
}
/// @returns The total size of the block in bytes, including the header.
- size_t outer_size() const { return next_ & ~3u; }
+ size_t outer_size() const { return next_ & SIZE_MASK; }
/// @returns The number of usable bytes inside the block.
size_t inner_size() const { return outer_size() - BLOCK_OVERHEAD; }
@@ -215,7 +220,7 @@ class Block {
/// Indicates whether the block is in use.
///
/// @returns `true` if the block is in use or `false` if not.
- bool used() const { return next_ & 1; }
+ bool used() const { return next_ & USED_MASK; }
/// Indicates whether this block is the last block or not (i.e. whether
/// `next()` points to a valid block or not). This is needed because
@@ -223,19 +228,19 @@ class Block {
/// block there or not.
///
/// @returns `true` is this is the last block or `false` if not.
- bool last() const { return next_ & 2; }
+ bool last() const { return next_ & LAST_MASK; }
/// Marks this block as in use.
- void mark_used() { next_ |= 1; }
+ void mark_used() { next_ |= USED_MASK; }
/// Marks this block as free.
- void mark_free() { next_ &= ~1u; }
+ void mark_free() { next_ &= ~USED_MASK; }
/// Marks this block as the last one in the chain.
- constexpr void mark_last() { next_ |= 2; }
+ constexpr void mark_last() { next_ |= LAST_MASK; }
/// Clears the last bit from this block.
- void clear_last() { next_ &= ~2u; }
+ void clear_last() { next_ &= ~LAST_MASK; }
/// @brief Checks if a block is valid.
///
More information about the libc-commits
mailing list