[compiler-rt] aff7348 - tsan: increase dense slab alloc capacity
Dmitry Vyukov via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 28 22:34:56 PDT 2021
Author: Dmitry Vyukov
Date: 2021-04-29T07:34:50+02:00
New Revision: aff73487c986b519aac1e0b7fee6d7bef72e4002
URL: https://github.com/llvm/llvm-project/commit/aff73487c986b519aac1e0b7fee6d7bef72e4002
DIFF: https://github.com/llvm/llvm-project/commit/aff73487c986b519aac1e0b7fee6d7bef72e4002.diff
LOG: tsan: increase dense slab alloc capacity
We've got a user report about heap block allocator overflow.
Bump the L1 capacity of all dense slab allocators to maximum
and be careful to not page the whole L1 array in from .bss.
If OS uses huge pages, this still may cause a limited RSS increase
due to boundary huge pages, but avoiding that looks hard.
Reviewed By: vitalybuka
Differential Revision: https://reviews.llvm.org/D101161
Added:
Modified:
compiler-rt/lib/tsan/rtl/tsan_clock.h
compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h
compiler-rt/lib/tsan/rtl/tsan_rtl.cpp
compiler-rt/lib/tsan/rtl/tsan_sync.cpp
compiler-rt/lib/tsan/rtl/tsan_sync.h
compiler-rt/lib/tsan/tests/unit/CMakeLists.txt
compiler-rt/lib/tsan/tests/unit/tsan_dense_alloc_test.cpp
Removed:
################################################################################
diff --git a/compiler-rt/lib/tsan/rtl/tsan_clock.h b/compiler-rt/lib/tsan/rtl/tsan_clock.h
index 736cdae06ba21..baeb53c2e289f 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_clock.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_clock.h
@@ -17,7 +17,7 @@
namespace __tsan {
-typedef DenseSlabAlloc<ClockBlock, 1<<16, 1<<10> ClockAlloc;
+typedef DenseSlabAlloc<ClockBlock, 1 << 22, 1 << 10> ClockAlloc;
typedef DenseSlabAllocCache ClockCache;
// The clock that lives in sync variables (mutexes, atomics, etc).
diff --git a/compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h b/compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h
index 64fc50e95c25f..6c89e40598042 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h
@@ -29,28 +29,40 @@ class DenseSlabAllocCache {
typedef u32 IndexT;
uptr pos;
IndexT cache[kSize];
- template<typename T, uptr kL1Size, uptr kL2Size> friend class DenseSlabAlloc;
+ template <typename, uptr, uptr, u64>
+ friend class DenseSlabAlloc;
};
-template<typename T, uptr kL1Size, uptr kL2Size>
+template <typename T, uptr kL1Size, uptr kL2Size, u64 kReserved = 0>
class DenseSlabAlloc {
public:
typedef DenseSlabAllocCache Cache;
typedef typename Cache::IndexT IndexT;
- explicit DenseSlabAlloc(const char *name) {
- // Check that kL1Size and kL2Size are sane.
- CHECK_EQ(kL1Size & (kL1Size - 1), 0);
- CHECK_EQ(kL2Size & (kL2Size - 1), 0);
- CHECK_GE(1ull << (sizeof(IndexT) * 8), kL1Size * kL2Size);
- // Check that it makes sense to use the dense alloc.
- CHECK_GE(sizeof(T), sizeof(IndexT));
- internal_memset(map_, 0, sizeof(map_));
+ static_assert((kL1Size & (kL1Size - 1)) == 0,
+ "kL1Size must be a power-of-two");
+ static_assert((kL2Size & (kL2Size - 1)) == 0,
+ "kL2Size must be a power-of-two");
+ static_assert((kL1Size * kL2Size) <= (1ull << (sizeof(IndexT) * 8)),
+ "kL1Size/kL2Size are too large");
+ static_assert(((kL1Size * kL2Size - 1) & kReserved) == 0,
+ "reserved bits don't fit");
+ static_assert(sizeof(T) > sizeof(IndexT),
+ "it doesn't make sense to use dense alloc");
+
+ explicit DenseSlabAlloc(LinkerInitialized, const char *name) {
freelist_ = 0;
fillpos_ = 0;
name_ = name;
}
+ explicit DenseSlabAlloc(const char *name)
+ : DenseSlabAlloc(LINKER_INITIALIZED, name) {
+ // It can be very large.
+ // Don't page it in for linker initialized objects.
+ internal_memset(map_, 0, sizeof(map_));
+ }
+
~DenseSlabAlloc() {
for (uptr i = 0; i < kL1Size; i++) {
if (map_[i] != 0)
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp
index ed6cc83450d90..8b49012b19501 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp
@@ -114,17 +114,17 @@ static const u32 kThreadQuarantineSize = 64;
#endif
Context::Context()
- : initialized()
- , report_mtx(MutexTypeReport, StatMtxReport)
- , nreported()
- , nmissed_expected()
- , thread_registry(new(thread_registry_placeholder) ThreadRegistry(
- CreateThreadContext, kMaxTid, kThreadQuarantineSize, kMaxTidReuse))
- , racy_mtx(MutexTypeRacy, StatMtxRacy)
- , racy_stacks()
- , racy_addresses()
- , fired_suppressions_mtx(MutexTypeFired, StatMtxFired)
- , clock_alloc("clock allocator") {
+ : initialized(),
+ report_mtx(MutexTypeReport, StatMtxReport),
+ nreported(),
+ nmissed_expected(),
+ thread_registry(new (thread_registry_placeholder) ThreadRegistry(
+ CreateThreadContext, kMaxTid, kThreadQuarantineSize, kMaxTidReuse)),
+ racy_mtx(MutexTypeRacy, StatMtxRacy),
+ racy_stacks(),
+ racy_addresses(),
+ fired_suppressions_mtx(MutexTypeFired, StatMtxFired),
+ clock_alloc(LINKER_INITIALIZED, "clock allocator") {
fired_suppressions.reserve(8);
}
diff --git a/compiler-rt/lib/tsan/rtl/tsan_sync.cpp b/compiler-rt/lib/tsan/rtl/tsan_sync.cpp
index 17ddd50f1284c..ba24f98ae9f6f 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_sync.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_sync.cpp
@@ -53,8 +53,8 @@ void SyncVar::Reset(Processor *proc) {
}
MetaMap::MetaMap()
- : block_alloc_("heap block allocator")
- , sync_alloc_("sync allocator") {
+ : block_alloc_(LINKER_INITIALIZED, "heap block allocator"),
+ sync_alloc_(LINKER_INITIALIZED, "sync allocator") {
atomic_store(&uid_gen_, 0, memory_order_relaxed);
}
diff --git a/compiler-rt/lib/tsan/rtl/tsan_sync.h b/compiler-rt/lib/tsan/rtl/tsan_sync.h
index 47f2739d8de5a..93d353e4fb450 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_sync.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_sync.h
@@ -130,8 +130,8 @@ class MetaMap {
static const u32 kFlagMask = 3u << 30;
static const u32 kFlagBlock = 1u << 30;
static const u32 kFlagSync = 2u << 30;
- typedef DenseSlabAlloc<MBlock, 1<<16, 1<<12> BlockAlloc;
- typedef DenseSlabAlloc<SyncVar, 1<<16, 1<<10> SyncAlloc;
+ typedef DenseSlabAlloc<MBlock, 1 << 18, 1 << 12, kFlagMask> BlockAlloc;
+ typedef DenseSlabAlloc<SyncVar, 1 << 20, 1 << 10, kFlagMask> SyncAlloc;
BlockAlloc block_alloc_;
SyncAlloc sync_alloc_;
atomic_uint64_t uid_gen_;
diff --git a/compiler-rt/lib/tsan/tests/unit/CMakeLists.txt b/compiler-rt/lib/tsan/tests/unit/CMakeLists.txt
index 79e334a2c6606..a2ada411db524 100644
--- a/compiler-rt/lib/tsan/tests/unit/CMakeLists.txt
+++ b/compiler-rt/lib/tsan/tests/unit/CMakeLists.txt
@@ -1,5 +1,6 @@
set(TSAN_UNIT_TEST_SOURCES
tsan_clock_test.cpp
+ tsan_dense_alloc_test.cpp
tsan_flags_test.cpp
tsan_mman_test.cpp
tsan_mutex_test.cpp
diff --git a/compiler-rt/lib/tsan/tests/unit/tsan_dense_alloc_test.cpp b/compiler-rt/lib/tsan/tests/unit/tsan_dense_alloc_test.cpp
index 05045622881db..02dddda8b3842 100644
--- a/compiler-rt/lib/tsan/tests/unit/tsan_dense_alloc_test.cpp
+++ b/compiler-rt/lib/tsan/tests/unit/tsan_dense_alloc_test.cpp
@@ -21,28 +21,29 @@
namespace __tsan {
TEST(DenseSlabAlloc, Basic) {
- typedef DenseSlabAlloc<int, 128, 128> Alloc;
+ typedef u64 T;
+ typedef DenseSlabAlloc<T, 128, 128> Alloc;
typedef Alloc::Cache Cache;
typedef Alloc::IndexT IndexT;
- const int N = 1000;
+ const T N = 1000;
- Alloc alloc;
+ Alloc alloc("test");
Cache cache;
alloc.InitCache(&cache);
IndexT blocks[N];
for (int ntry = 0; ntry < 3; ntry++) {
- for (int i = 0; i < N; i++) {
+ for (T i = 0; i < N; i++) {
IndexT idx = alloc.Alloc(&cache);
blocks[i] = idx;
EXPECT_NE(idx, 0U);
- int *v = alloc.Map(idx);
+ T *v = alloc.Map(idx);
*v = i;
}
- for (int i = 0; i < N; i++) {
+ for (T i = 0; i < N; i++) {
IndexT idx = blocks[i];
- int *v = alloc.Map(idx);
+ T *v = alloc.Map(idx);
EXPECT_EQ(*v, i);
alloc.Free(&cache, idx);
}
More information about the llvm-commits
mailing list