[compiler-rt] [llvm] [Memprof] Adds instrumentation support for memprof with histograms. (PR #100834)
Matthew Weingarten via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 29 12:39:10 PDT 2024
https://github.com/mattweingarten updated https://github.com/llvm/llvm-project/pull/100834
>From 68725fd958ec6d480eab20faca4612148aeb1fdf Mon Sep 17 00:00:00 2001
From: Matthew Weingarten <matt at weingarten.org>
Date: Mon, 22 Jul 2024 22:56:58 +0000
Subject: [PATCH 1/3] [Memprof] Adds instrumentation support for running
memprof with histograms.
This patch allows running `-fmemory-profile` without `-memprof-use-callbacks`. This significantly increases
performance of the instrumented binary.
Additionally, changes the `HISTOGRAM_GRANULARITY` from 8U to 8ULL to correct shadow address translation.
---
compiler-rt/lib/memprof/memprof_mapping.h | 2 +-
.../Instrumentation/MemProfiler.cpp | 52 +++++++++++++------
2 files changed, 38 insertions(+), 16 deletions(-)
diff --git a/compiler-rt/lib/memprof/memprof_mapping.h b/compiler-rt/lib/memprof/memprof_mapping.h
index fef8acfcfc921..6da385ab3d6e2 100644
--- a/compiler-rt/lib/memprof/memprof_mapping.h
+++ b/compiler-rt/lib/memprof/memprof_mapping.h
@@ -55,7 +55,7 @@ extern uptr kHighMemEnd; // Initialized in __memprof_init.
// computed by summing up all individual 1 byte counters. This can incur an
// accuracy penalty.
-#define HISTOGRAM_GRANULARITY 8U
+#define HISTOGRAM_GRANULARITY 8ULL
#define HISTOGRAM_MAX_COUNTER 255U
diff --git a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
index 2c5d749d4a67a..0e4d21671d2a7 100644
--- a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
@@ -61,6 +61,9 @@ constexpr int LLVM_MEM_PROFILER_VERSION = 1;
// Size of memory mapped to a single shadow location.
constexpr uint64_t DefaultMemGranularity = 64;
+// Size of memory mapped to a single histogram bucket.
+constexpr uint64_t HistogramGranularity = 8;
+
// Scale from granularity down to shadow size.
constexpr uint64_t DefaultShadowScale = 3;
@@ -192,7 +195,8 @@ namespace {
struct ShadowMapping {
ShadowMapping() {
Scale = ClMappingScale;
- Granularity = ClMappingGranularity;
+ // Histogram Granularity is always 8U for now.
+ Granularity = ClHistogram ? HistogramGranularity : ClMappingGranularity;
Mask = ~(Granularity - 1);
}
@@ -288,10 +292,6 @@ ModuleMemProfilerPass::ModuleMemProfilerPass() = default;
PreservedAnalyses ModuleMemProfilerPass::run(Module &M,
AnalysisManager<Module> &AM) {
- assert((!ClHistogram || (ClHistogram && ClUseCalls)) &&
- "Cannot use -memprof-histogram without Callbacks. Set "
- "memprof-use-callbacks");
-
ModuleMemProfiler Profiler(M);
if (Profiler.instrumentModule(M))
return PreservedAnalyses::none();
@@ -489,16 +489,38 @@ void MemProfiler::instrumentAddress(Instruction *OrigIns,
return;
}
- // Create an inline sequence to compute shadow location, and increment the
- // value by one.
- Type *ShadowTy = Type::getInt64Ty(*C);
- Type *ShadowPtrTy = PointerType::get(ShadowTy, 0);
- Value *ShadowPtr = memToShadow(AddrLong, IRB);
- Value *ShadowAddr = IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy);
- Value *ShadowValue = IRB.CreateLoad(ShadowTy, ShadowAddr);
- Value *Inc = ConstantInt::get(Type::getInt64Ty(*C), 1);
- ShadowValue = IRB.CreateAdd(ShadowValue, Inc);
- IRB.CreateStore(ShadowValue, ShadowAddr);
+ if (ClHistogram) {
+ Type *ShadowTy = Type::getInt8Ty(*C);
+ Type *ShadowPtrTy = PointerType::get(ShadowTy, 0);
+ Value *Shadow = IRB.CreateAnd(AddrLong, Mapping.Mask);
+ Shadow = IRB.CreateLShr(Shadow, Mapping.Scale);
+ assert(DynamicShadowOffset);
+ Value *ShadowInteger = IRB.CreateAdd(Shadow, DynamicShadowOffset);
+
+ Value *ShadowAddr = IRB.CreateIntToPtr(ShadowInteger, ShadowPtrTy);
+ Value *ShadowValue = IRB.CreateLoad(ShadowTy, ShadowAddr);
+ Value *MaxCount = ConstantInt::get(Type::getInt8Ty(*C), 255);
+ Value *Cmp = IRB.CreateICmpULT(ShadowValue, MaxCount);
+ // Insert Basic block that increments histogram bucket if value is less
+ // than 255.
+ Instruction *IncBlock =
+ SplitBlockAndInsertIfThen(Cmp, InsertBefore, /*Unreachable=*/false);
+ IRB.SetInsertPoint(IncBlock);
+ Value *Inc = ConstantInt::get(Type::getInt8Ty(*C), 1);
+ ShadowValue = IRB.CreateAdd(ShadowValue, Inc);
+ IRB.CreateStore(ShadowValue, ShadowAddr);
+ } else {
+ // Create an inline sequence to compute shadow location, and increment the
+ // value by one.
+ Type *ShadowTy = Type::getInt64Ty(*C);
+ Type *ShadowPtrTy = PointerType::get(ShadowTy, 0);
+ Value *ShadowPtr = memToShadow(AddrLong, IRB);
+ Value *ShadowAddr = IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy);
+ Value *ShadowValue = IRB.CreateLoad(ShadowTy, ShadowAddr);
+ Value *Inc = ConstantInt::get(Type::getInt64Ty(*C), 1);
+ ShadowValue = IRB.CreateAdd(ShadowValue, Inc);
+ IRB.CreateStore(ShadowValue, ShadowAddr);
+ }
}
// Create the variable for the profile file name.
>From 3bf990406fb7b4afea2ff0a368c56b3a9d545664 Mon Sep 17 00:00:00 2001
From: Matthew Weingarten <matt at weingarten.org>
Date: Sun, 28 Jul 2024 23:14:54 +0000
Subject: [PATCH 2/3] fixup! [Memprof] Adds instrumentation support for running
memprof with histograms.
---
.../Instrumentation/MemProfiler.cpp | 36 ++++--------
.../HeapProfiler/basic-histogram.ll | 57 +++++++++++++++++++
2 files changed, 67 insertions(+), 26 deletions(-)
create mode 100644 llvm/test/Instrumentation/HeapProfiler/basic-histogram.ll
diff --git a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
index 0e4d21671d2a7..8e8eb85fa729c 100644
--- a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
@@ -195,7 +195,6 @@ namespace {
struct ShadowMapping {
ShadowMapping() {
Scale = ClMappingScale;
- // Histogram Granularity is always 8U for now.
Granularity = ClHistogram ? HistogramGranularity : ClMappingGranularity;
Mask = ~(Granularity - 1);
}
@@ -489,38 +488,23 @@ void MemProfiler::instrumentAddress(Instruction *OrigIns,
return;
}
+ Type *ShadowTy = ClHistogram ? Type::getInt8Ty(*C) : Type::getInt64Ty(*C);
+ Type *ShadowPtrTy = PointerType::get(ShadowTy, 0);
+
+ Value *ShadowPtr = memToShadow(AddrLong, IRB);
+ Value *ShadowAddr = IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy);
+ Value *ShadowValue = IRB.CreateLoad(ShadowTy, ShadowAddr);
+ // If we are profiling with histograms, add overflow protection at 255.
if (ClHistogram) {
- Type *ShadowTy = Type::getInt8Ty(*C);
- Type *ShadowPtrTy = PointerType::get(ShadowTy, 0);
- Value *Shadow = IRB.CreateAnd(AddrLong, Mapping.Mask);
- Shadow = IRB.CreateLShr(Shadow, Mapping.Scale);
- assert(DynamicShadowOffset);
- Value *ShadowInteger = IRB.CreateAdd(Shadow, DynamicShadowOffset);
-
- Value *ShadowAddr = IRB.CreateIntToPtr(ShadowInteger, ShadowPtrTy);
- Value *ShadowValue = IRB.CreateLoad(ShadowTy, ShadowAddr);
Value *MaxCount = ConstantInt::get(Type::getInt8Ty(*C), 255);
Value *Cmp = IRB.CreateICmpULT(ShadowValue, MaxCount);
- // Insert Basic block that increments histogram bucket if value is less
- // than 255.
Instruction *IncBlock =
SplitBlockAndInsertIfThen(Cmp, InsertBefore, /*Unreachable=*/false);
IRB.SetInsertPoint(IncBlock);
- Value *Inc = ConstantInt::get(Type::getInt8Ty(*C), 1);
- ShadowValue = IRB.CreateAdd(ShadowValue, Inc);
- IRB.CreateStore(ShadowValue, ShadowAddr);
- } else {
- // Create an inline sequence to compute shadow location, and increment the
- // value by one.
- Type *ShadowTy = Type::getInt64Ty(*C);
- Type *ShadowPtrTy = PointerType::get(ShadowTy, 0);
- Value *ShadowPtr = memToShadow(AddrLong, IRB);
- Value *ShadowAddr = IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy);
- Value *ShadowValue = IRB.CreateLoad(ShadowTy, ShadowAddr);
- Value *Inc = ConstantInt::get(Type::getInt64Ty(*C), 1);
- ShadowValue = IRB.CreateAdd(ShadowValue, Inc);
- IRB.CreateStore(ShadowValue, ShadowAddr);
}
+ Value *Inc = ConstantInt::get(ShadowTy, 1);
+ ShadowValue = IRB.CreateAdd(ShadowValue, Inc);
+ IRB.CreateStore(ShadowValue, ShadowAddr);
}
// Create the variable for the profile file name.
diff --git a/llvm/test/Instrumentation/HeapProfiler/basic-histogram.ll b/llvm/test/Instrumentation/HeapProfiler/basic-histogram.ll
new file mode 100644
index 0000000000000..c7ff129e5b4c4
--- /dev/null
+++ b/llvm/test/Instrumentation/HeapProfiler/basic-histogram.ll
@@ -0,0 +1,57 @@
+; Test basic memory profiler instrumentation with histograms.
+;
+; RUN: opt < %s -passes='function(memprof),memprof-module' -memprof-histogram -S | FileCheck --check-prefixes=CHECK,CHECK-S3 %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: @llvm.used = appending global [1 x ptr] [ptr @memprof.module_ctor]
+; CHECK: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 1, ptr @memprof.module_ctor, ptr null }]
+
+define i32 @test_load(ptr %a) {
+entry:
+ %tmp1 = load i32, ptr %a, align 4
+ ret i32 %tmp1
+}
+; CHECK-LABEL: @test_load
+; CHECK: %[[SHADOW_OFFSET:[^ ]*]] = load i64, ptr @__memprof_shadow_memory_dynamic_address
+; CHECK-NEXT: %[[LOAD_ADDR:[^ ]*]] = ptrtoint ptr %a to i64
+; CHECK-NEXT: %[[MASKED_ADDR:[^ ]*]] = and i64 %[[LOAD_ADDR]], -8
+; CHECK-S3-NEXT: %[[SHIFTED_ADDR:[^ ]*]] = lshr i64 %[[MASKED_ADDR]], 3
+; CHECK-NEXT: add i64 %[[SHIFTED_ADDR]], %[[SHADOW_OFFSET]]
+; CHECK-NEXT: %[[LOAD_SHADOW_PTR:[^ ]*]] = inttoptr
+; CHECK-NEXT: %[[LOAD_SHADOW:[^ ]*]] = load i8, ptr %[[LOAD_SHADOW_PTR]]
+; CHECK-NEXT: %[[ICMP_MAX_COUNT:[^ ]*]] = icmp ult i8 %[[LOAD_SHADOW]], -1
+; CHECK-NEXT: br i1 %[[ICMP_MAX_COUNT]], label %[[INC_LABEL:[^ ]*]], label %[[ELSE_LABEL:[^ ]*]]
+; CHECK: [[INC_LABEL]]:
+; CHECK-NEXT: %[[NEW_SHADOW:[^ ]*]] = add i8 %[[LOAD_SHADOW]], 1
+; CHECK-NEXT: store i8 %[[NEW_SHADOW]], ptr %[[LOAD_SHADOW_PTR]]
+; CHECK-NEXT: br label %[[ELSE_LABEL]]
+; The actual load.
+; CHECK: [[ELSE_LABEL]]:
+; CHECK-NEXT: %tmp1 = load i32, ptr %a
+; CHECK-NEXT: ret i32 %tmp1
+
+define void @test_store(ptr %a) {
+entry:
+ store i32 42, ptr %a, align 4
+ ret void
+}
+; CHECK-LABEL: @test_store
+; CHECK: %[[SHADOW_OFFSET:[^ ]*]] = load i64, ptr @__memprof_shadow_memory_dynamic_address
+; CHECK-NEXT: %[[LOAD_ADDR:[^ ]*]] = ptrtoint ptr %a to i64
+; CHECK-NEXT: %[[MASKED_ADDR:[^ ]*]] = and i64 %[[LOAD_ADDR]], -8
+; CHECK-S3-NEXT: %[[SHIFTED_ADDR:[^ ]*]] = lshr i64 %[[MASKED_ADDR]], 3
+; CHECK-NEXT: add i64 %[[SHIFTED_ADDR]], %[[SHADOW_OFFSET]]
+; CHECK-NEXT: %[[STORE_SHADOW_PTR:[^ ]*]] = inttoptr
+; CHECK-NEXT: %[[STORE_SHADOW:[^ ]*]] = load i8, ptr %[[STORE_SHADOW_PTR]]
+; CHECK-NEXT: %[[ICMP_MAX_COUNT:[^ ]*]] = icmp ult i8 %[[STORE_SHADOW]], -1
+; CHECK-NEXT: br i1 %[[ICMP_MAX_COUNT]], label %[[INC_LABEL:[^ ]*]], label %[[ELSE_LABEL:[^ ]*]]
+; CHECK: [[INC_LABEL]]:
+; CHECK-NEXT: %[[NEW_SHADOW:[^ ]*]] = add i8 %[[STORE_SHADOW]], 1
+; CHECK-NEXT: store i8 %[[NEW_SHADOW]], ptr %[[STORE_SHADOW_PTR]]
+; CHECK-NEXT: br label %[[ELSE_LABEL]]
+; The actual store.
+; CHECK: [[ELSE_LABEL]]:
+; CHECK-NEXT: store i32 42, ptr %a, align 4
+; CHECK-NEXT: ret void
\ No newline at end of file
>From 07a5c6518eb19181a9f82699e9958ddad76fde6a Mon Sep 17 00:00:00 2001
From: Matthew Weingarten <matt at weingarten.org>
Date: Mon, 29 Jul 2024 19:38:23 +0000
Subject: [PATCH 3/3] fixup! fixup! [Memprof] Adds instrumentation support for
running memprof with histograms.
---
llvm/lib/Transforms/Instrumentation/MemProfiler.cpp | 2 ++
1 file changed, 2 insertions(+)
diff --git a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
index 8e8eb85fa729c..445bf0bb26cc4 100644
--- a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
@@ -279,6 +279,8 @@ MemProfilerPass::MemProfilerPass() = default;
PreservedAnalyses MemProfilerPass::run(Function &F,
AnalysisManager<Function> &AM) {
+ assert((!ClHistogram || ClMappingGranularity == DefaultMemGranularity) &&
+ "Memprof with histogram only supports default mapping granularity");
Module &M = *F.getParent();
MemProfiler Profiler(M);
if (Profiler.instrumentFunction(F))
More information about the llvm-commits
mailing list