[llvm] [ADT][ProfileData] Introduce SortedVectorMap and switch CallTargetMap to it (PR #215733)
Kazu Hirata via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 11 23:14:32 PDT 2026
https://github.com/kazutakahirata created https://github.com/llvm/llvm-project/pull/215733
This patch introduces SortedVectorMap, a map implementation backed by
a sorted SmallVector, and switches SampleRecord::CallTargetMap from
DenseMap<FunctionId, uint64_t> to
SortedVectorMap<FunctionId, uint64_t, 0>.
Commit 3746f3e4d612 previously changed CallTargetMap from
std::unordered_map<FunctionId, uint64_t> to DenseMap. However, greater
than 97% of CallTargetMap instances have no more than one
element. When storing a single callee, the DenseMap incurs over 60x
memory overhead compared to a single std::pair<FunctionId, uint64_t>
(allocating a 64-bucket table and bit vector of ~1.5 KB vs 24
bytes). Since we instantiate CallTargetMap for every call site, this
~1.5 KB table floor per record adds up to gigabytes of wasted heap
memory across a large profile.
SortedVectorMap keeps key-value pairs in contiguous memory ordered by
key and uses std::lower_bound for lookups. Configuring N = 0 inline
capacity keeps empty maps at 16 bytes and allocates exact payload
vectors on demand. Beyond CallTargetMap, SortedVectorMap will also be
useful for replacing several std::map instances across the sample
profile implementation.
Empirical benchmark results:
Profile merging:
Metric Before After Change
Max RSS 85.1 GiB 21.8 GiB -74.4%
Wall-clock time 76.6 s 38.6 s -49.6%
Minor page faults 8.76M 48.9k -99.4%
Compilation speed: About 0.05% speedup on about 100,000 compilations
of different C++ source files.
Alternative containers such as std::unordered_map and
SmallDenseMap<..., 2> provide smaller memory savings and/or regress
compilation speed.
Assisted-by: Antigravity
>From ee304f6f5d5c1adbfe78028dc208e4db520d2178 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu at google.com>
Date: Tue, 11 Aug 2026 21:36:30 -0700
Subject: [PATCH] [ADT][ProfileData] Introduce SortedVectorMap and switch
CallTargetMap to it
This patch introduces SortedVectorMap, a map implementation backed by
a sorted SmallVector, and switches SampleRecord::CallTargetMap from
DenseMap<FunctionId, uint64_t> to
SortedVectorMap<FunctionId, uint64_t, 0>.
Commit 3746f3e4d612 previously changed CallTargetMap from
std::unordered_map<FunctionId, uint64_t> to DenseMap. However, greater
than 97% of CallTargetMap instances have no more than one
element. When storing a single callee, the DenseMap incurs over 60x
memory overhead compared to a single std::pair<FunctionId, uint64_t>
(allocating a 64-bucket table and bit vector of ~1.5 KB vs 24
bytes). Since we instantiate CallTargetMap for every call site, this
~1.5 KB table floor per record adds up to gigabytes of wasted heap
memory across a large profile.
SortedVectorMap keeps key-value pairs in contiguous memory ordered by
key and uses std::lower_bound for lookups. Configuring N = 0 inline
capacity keeps empty maps at 16 bytes and allocates exact payload
vectors on demand. Beyond CallTargetMap, SortedVectorMap will also be
useful for replacing several std::map instances across the sample
profile implementation.
Empirical benchmark results:
Profile merging:
Metric Before After Change
Max RSS 85.1 GiB 21.8 GiB -74.4%
Wall-clock time 76.6 s 38.6 s -49.6%
Minor page faults 8.76M 48.9k -99.4%
Compilation speed: About 0.05% speedup on about 100,000 compilations
of different C++ source files.
Alternative containers such as std::unordered_map and
SmallDenseMap<..., 2> provide smaller memory savings and/or regress
compilation speed.
Assisted-by: Antigravity
---
llvm/include/llvm/ADT/SortedVectorMap.h | 124 +++++++++++++++++++++
llvm/include/llvm/ProfileData/SampleProf.h | 3 +-
llvm/unittests/ADT/SortedVectorMapTest.cpp | 91 +++++++++++++++
3 files changed, 217 insertions(+), 1 deletion(-)
create mode 100644 llvm/include/llvm/ADT/SortedVectorMap.h
create mode 100644 llvm/unittests/ADT/SortedVectorMapTest.cpp
diff --git a/llvm/include/llvm/ADT/SortedVectorMap.h b/llvm/include/llvm/ADT/SortedVectorMap.h
new file mode 100644
index 0000000000000..486880871afe0
--- /dev/null
+++ b/llvm/include/llvm/ADT/SortedVectorMap.h
@@ -0,0 +1,124 @@
+//===- llvm/ADT/SortedVectorMap.h - Map backed by SmallVector *- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file implements a map backed by a sorted SmallVector. It provides a
+/// std::map-like interface with binary search lookup while maintaining
+/// contiguous memory layout and dense L1 cache density.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ADT_SORTEDVECTORMAP_H
+#define LLVM_ADT_SORTEDVECTORMAP_H
+
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Compiler.h"
+#include <functional>
+#include <utility>
+
+namespace llvm {
+
+/// A map implementation backed by a sorted SmallVector.
+/// Key-value pairs are stored in contiguous memory ordered by \p KeyCompare.
+template <typename KeyT, typename ValueT, unsigned N = 0,
+ typename KeyCompare = std::less<KeyT>>
+class SortedVectorMap {
+public:
+ using key_type = KeyT;
+ using mapped_type = ValueT;
+ using value_type = std::pair<KeyT, ValueT>;
+ using VectorType = SmallVector<value_type, N>;
+ using size_type = typename VectorType::size_type;
+
+ using iterator = typename VectorType::iterator;
+ using const_iterator = typename VectorType::const_iterator;
+
+private:
+ VectorType Vector;
+ LLVM_NO_UNIQUE_ADDRESS KeyCompare Comp;
+
+ template <typename K1, typename K2>
+ bool is_equal(const K1 &A, const K2 &B) const {
+ return !Comp(A, B) && !Comp(B, A);
+ }
+
+ template <typename K> const_iterator lower_bound(const K &Key) const {
+ return llvm::lower_bound(Vector, Key,
+ [this](const value_type &E, const K &KeyVal) {
+ return Comp(E.first, KeyVal);
+ });
+ }
+
+ template <typename K>
+ std::pair<const_iterator, bool> find_or_insert_location(const K &Key) const {
+ if (!Vector.empty() && Comp(Vector.back().first, Key))
+ return {Vector.end(), false};
+ auto It = lower_bound(Key);
+ bool Found = (It != Vector.end() && is_equal(Key, It->first));
+ return {It, Found};
+ }
+
+ template <typename K>
+ std::pair<iterator, bool> find_or_insert_location(const K &Key) {
+ auto [ConstIt, Found] = std::as_const(*this).find_or_insert_location(Key);
+ return {Vector.begin() + (ConstIt - Vector.begin()), Found};
+ }
+
+public:
+ SortedVectorMap() = default;
+
+ // Iterators
+ iterator begin() { return Vector.begin(); }
+ iterator end() { return Vector.end(); }
+ const_iterator begin() const { return Vector.begin(); }
+ const_iterator end() const { return Vector.end(); }
+
+ // Capacity
+ [[nodiscard]] bool empty() const { return Vector.empty(); }
+ size_type size() const { return Vector.size(); }
+ size_type capacity() const { return Vector.capacity(); }
+ void reserve(size_type Cap) { Vector.reserve(Cap); }
+
+ // Element Access & Lookups
+
+ template <typename K> const_iterator find(const K &Key) const {
+ auto [It, Found] = find_or_insert_location(Key);
+ return Found ? It : Vector.end();
+ }
+
+ template <typename K> iterator find(const K &Key) {
+ auto [It, Found] = find_or_insert_location(Key);
+ return Found ? It : Vector.end();
+ }
+
+ ValueT &operator[](const KeyT &Key) {
+ auto [It, Found] = find_or_insert_location(Key);
+ if (Found)
+ return It->second;
+ return Vector.insert(It, value_type(Key, ValueT()))->second;
+ }
+
+ ValueT &operator[](KeyT &&Key) {
+ auto [It, Found] = find_or_insert_location(Key);
+ if (Found)
+ return It->second;
+ return Vector.insert(It, value_type(std::move(Key), ValueT()))->second;
+ }
+
+ iterator erase(iterator Pos) { return Vector.erase(Pos); }
+ iterator erase(const_iterator Pos) { return Vector.erase(Pos); }
+
+ bool operator==(const SortedVectorMap &Other) const {
+ return Vector == Other.Vector;
+ }
+};
+
+} // namespace llvm
+
+#endif // LLVM_ADT_SORTEDVECTORMAP_H
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index 3d69b7711588a..4e484a401fd0a 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -19,6 +19,7 @@
#include "llvm/ADT/Eytzinger.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/SortedVectorMap.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/IR/Function.h"
@@ -403,7 +404,7 @@ class SampleRecord {
};
using SortedCallTargetSet = SmallVector<CallTarget>;
- using CallTargetMap = DenseMap<FunctionId, uint64_t>;
+ using CallTargetMap = SortedVectorMap<FunctionId, uint64_t, 0>;
SampleRecord() = default;
/// Increment the number of samples for this record by \p S.
diff --git a/llvm/unittests/ADT/SortedVectorMapTest.cpp b/llvm/unittests/ADT/SortedVectorMapTest.cpp
new file mode 100644
index 0000000000000..ce7c5f66604f2
--- /dev/null
+++ b/llvm/unittests/ADT/SortedVectorMapTest.cpp
@@ -0,0 +1,91 @@
+//===- llvm/unittest/ADT/SortedVectorMapTest.cpp -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/SortedVectorMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "gtest/gtest.h"
+#include <string>
+
+using namespace llvm;
+
+namespace {
+
+TEST(SortedVectorMapTest, BasicOperations) {
+ SortedVectorMap<int, std::string> Map;
+
+ EXPECT_TRUE(Map.empty());
+ EXPECT_EQ(Map.size(), 0u);
+
+ Map[5] = "five";
+ Map[2] = "two";
+ Map[8] = "eight";
+
+ EXPECT_FALSE(Map.empty());
+ EXPECT_EQ(Map.size(), 3u);
+
+ EXPECT_EQ(Map[2], "two");
+ EXPECT_EQ(Map[5], "five");
+ EXPECT_EQ(Map[8], "eight");
+
+ // Verify elements are maintained in sorted key order
+ auto It = Map.begin();
+ EXPECT_EQ(It->first, 2);
+ EXPECT_EQ(It->second, "two");
+ ++It;
+ EXPECT_EQ(It->first, 5);
+ EXPECT_EQ(It->second, "five");
+ ++It;
+ EXPECT_EQ(It->first, 8);
+ EXPECT_EQ(It->second, "eight");
+ ++It;
+ EXPECT_EQ(It, Map.end());
+}
+
+TEST(SortedVectorMapTest, FindAndErase) {
+ SortedVectorMap<int, int> Map;
+ Map[10] = 100;
+ Map[20] = 200;
+ Map[30] = 300;
+
+ auto It = Map.find(20);
+ ASSERT_NE(It, Map.end());
+ EXPECT_EQ(It->second, 200);
+
+ EXPECT_EQ(Map.find(99), Map.end());
+
+ It = Map.erase(It);
+ EXPECT_EQ(Map.size(), 2u);
+ EXPECT_EQ(Map.find(20), Map.end());
+ ASSERT_NE(It, Map.end());
+ EXPECT_EQ(It->first, 30);
+}
+
+TEST(SortedVectorMapTest, EqualityOperator) {
+ SortedVectorMap<int, int> Map1;
+ SortedVectorMap<int, int> Map2;
+
+ Map1[1] = 10;
+ Map1[2] = 20;
+
+ Map2[2] = 20;
+ Map2[1] = 10;
+
+ EXPECT_EQ(Map1, Map2);
+}
+
+TEST(SortedVectorMapTest, ReserveAndCapacity) {
+ SortedVectorMap<int, int> Map;
+ EXPECT_EQ(Map.size(), 0u);
+ Map.reserve(50);
+ EXPECT_GE(Map.capacity(), 50u);
+ Map[1] = 10;
+ EXPECT_EQ(Map.size(), 1u);
+ EXPECT_GE(Map.capacity(), 50u);
+}
+
+} // namespace
More information about the llvm-commits
mailing list