[llvm] [InstrProf] Support conditional counter updates for integer counters (PR #109222)
Alan Zhao via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 19 11:22:45 PDT 2024
https://github.com/alanzhao1 updated https://github.com/llvm/llvm-project/pull/109222
>From be6ff5f7f4d75e710b5d056009b39f165074569e Mon Sep 17 00:00:00 2001
From: Alan Zhao <ayzhao at google.com>
Date: Wed, 18 Sep 2024 16:56:01 -0700
Subject: [PATCH 1/3] [InstrProf] Support conditional counter updates for
integer counters
In #102542. conditional counter updates were only implemented for
boolean counters. This PR implements conditional counter updates for
integer counters. When the flag -conditional-counter-update is set,
integer counters are now treated like boolean counters -
@llvm.instrprof.update now checks whether or not a counter is zero, and
if not, writes 1 to it.
The motiviation for this mode is that we can bring the benefits of
conditional single byte counter updates to languages whose frontends do
not support single byte counters (e.g. Rust).
---
.../Instrumentation/InstrProfiling.cpp | 16 +++-
.../conditional-counter-updates.ll | 91 +++++++++++++++++++
2 files changed, 105 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 014e049ed0d8e5..9d5394deda0856 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -1263,8 +1263,20 @@ void InstrLowerer::lowerIncrement(InstrProfIncrementInst *Inc) {
MaybeAlign(), AtomicOrdering::Monotonic);
} else {
Value *IncStep = Inc->getStep();
- Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
- auto *Count = Builder.CreateAdd(Load, Inc->getStep());
+ auto *CtrTy = IncStep->getType();
+ Value *Load = Builder.CreateLoad(CtrTy, Addr, "pgocount");
+ Value *Count;
+
+ if (ConditionalCounterUpdate) {
+ Instruction *SplitBefore = Inc->getNextNode();
+ Value *Cmp = Builder.CreateIsNull(Load, "pgocount.ifzero");
+ Instruction *ThenBranch =
+ SplitBlockAndInsertIfThen(Cmp, SplitBefore, false);
+ Builder.SetInsertPoint(ThenBranch);
+ Count = ConstantInt::get(CtrTy, 1);
+ } else {
+ Count = Builder.CreateAdd(Load, Inc->getStep());
+ }
auto *Store = Builder.CreateStore(Count, Addr);
if (isCounterPromotionEnabled())
PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
diff --git a/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll b/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll
index 998443ec0cd568..ebd510cfb5f08a 100644
--- a/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll
@@ -5,6 +5,9 @@ target triple = "x86_64-unknown-linux-gnu"
@__profn_foo = private constant [3 x i8] c"foo"
@__profn_bar = private constant [3 x i8] c"bar"
+ at __profn_fooint = private constant [6 x i8] c"fooint"
+ at __profn_barint = private constant [6 x i8] c"barint"
+
; CHECK-LABEL: define void @foo
; CHECK-NEXT: %pgocount = load i8, ptr @__profc_foo, align 1
; CHECK-NEXT: %pgocount.ifnonzero = icmp ne i8 %pgocount, 0
@@ -67,3 +70,91 @@ return: ; preds = %if.end, %if.then
%1 = load i32, ptr %retval, align 4
ret i32 %1
}
+
+; CHECK-LABEL: define void @fooint
+; CHECK-NEXT: %pgocount = load i64, ptr @__profc_fooint, align 8
+; CHECK-NEXT: %pgocount.ifzero = icmp eq i64 %pgocount, 0
+; CHECK-NEXT: br i1 %pgocount.ifzero, label %1, label %2
+
+; CHECK-LABEL: 1:
+; CHECK-NEXT: store i64 1, ptr @__profc_fooint, align 8
+; CHECK-NEXT: br label %2
+
+; CHECK-LABEL: 2:
+; CHECK-NEXT: ret void
+define void @fooint() {
+ call void @llvm.instrprof.increment(ptr @__profn_fooint, i64 0, i32 1, i32 0)
+ ret void
+}
+
+; CHECK-LABEL: define i32 @barint
+; CHECK-LABEL: entry:
+; CHECK-NEXT: %retval = alloca i32, align 4
+; CHECK-NEXT: %cond.addr = alloca i32, align 4
+; CHECK-NEXT: store i32 %cond, ptr %cond.addr, align 4
+; CHECK-NEXT: %pgocount = load i64, ptr @__profc_barint, align 8
+; CHECK-NEXT: %pgocount.ifzero = icmp eq i64 %pgocount, 0
+; CHECK-NEXT: br i1 %pgocount.ifzero, label %0, label %1
+
+; CHECK-LABEL: 0: ; preds = %entry
+; CHECK-NEXT: store i64 1, ptr @__profc_barint, align 8
+; CHECK-NEXT: br label %1
+
+; CHECK-LABEL: 1: ; preds = %entry, %0
+; CHECK-NEXT: %2 = load i32, ptr %cond.addr, align 4
+; CHECK-NEXT: %cmp = icmp slt i32 %2, 0
+; CHECK-NEXT: br i1 %cmp, label %if.then, label %if.end
+
+; CHECK-LABEL: if.then: ; preds = %1
+; CHECK-NEXT: %pgocount1 = load i64, ptr getelementptr inbounds ([3 x i64], ptr @__profc_barint, i32 0, i32 1), align 8
+; CHECK-NEXT: %pgocount.ifzero2 = icmp eq i64 %pgocount1, 0
+; CHECK-NEXT: br i1 %pgocount.ifzero2, label %3, label %4
+
+; CHECK-LABEL: 3: ; preds = %if.then
+; CHECK-NEXT: store i64 1, ptr getelementptr inbounds ([3 x i64], ptr @__profc_barint, i32 0, i32 1), align 8
+; CHECK-NEXT: br label %4
+;
+; CHECK-LABEL: 4: ; preds = %if.then, %3
+; CHECK-NEXT: store i32 -1, ptr %retval, align 4
+; CHECK-NEXT: br label %return
+;
+; CHECK-LABEL: if.end: ; preds = %1
+; CHECK-NEXT: %pgocount3 = load i64, ptr getelementptr inbounds ([3 x i64], ptr @__profc_barint, i32 0, i32 2), align 8
+; CHECK-NEXT: %pgocount.ifzero4 = icmp eq i64 %pgocount3, 0
+; CHECK-NEXT: br i1 %pgocount.ifzero4, label %5, label %6
+;
+; CHECK-LABEL: 5: ; preds = %if.end
+; CHECK-NEXT: store i64 1, ptr getelementptr inbounds ([3 x i64], ptr @__profc_barint, i32 0, i32 2), align 8
+; CHECK-NEXT: br label %6
+;
+; CHECK-LABEL: 6: ; preds = %if.end, %5
+; CHECK-NEXT: store i32 0, ptr %retval, align 4
+; CHECK-NEXT: br label %return
+;
+; CHECK-LABEL: return: ; preds = %6, %4
+; CHECK-NEXT: %7 = load i32, ptr %retval, align 4
+; CHECK-NEXT: ret i32 %7
+define i32 @barint(i32 %cond) #0 {
+entry:
+ %retval = alloca i32, align 4
+ %cond.addr = alloca i32, align 4
+ store i32 %cond, ptr %cond.addr, align 4
+ call void @llvm.instrprof.increment(ptr @__profn_barint, i64 0, i32 3, i32 0)
+ %0 = load i32, ptr %cond.addr, align 4
+ %cmp = icmp slt i32 %0, 0
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ call void @llvm.instrprof.increment(ptr @__profn_barint, i64 0, i32 3, i32 1)
+ store i32 -1, ptr %retval, align 4
+ br label %return
+
+if.end: ; preds = %entry
+ call void @llvm.instrprof.increment(ptr @__profn_barint, i64 0, i32 3, i32 2)
+ store i32 0, ptr %retval, align 4
+ br label %return
+
+return: ; preds = %if.end, %if.then
+ %1 = load i32, ptr %retval, align 4
+ ret i32 %1
+}
\ No newline at end of file
>From 21395013527fa38041cbed866737c5cc427f9026 Mon Sep 17 00:00:00 2001
From: Alan Zhao <ayzhao at google.com>
Date: Wed, 18 Sep 2024 17:12:43 -0700
Subject: [PATCH 2/3] spacing
---
llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp | 1 +
.../InstrProfiling/conditional-counter-updates.ll | 3 ++-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 9d5394deda0856..6f05835aa4cfbc 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -1277,6 +1277,7 @@ void InstrLowerer::lowerIncrement(InstrProfIncrementInst *Inc) {
} else {
Count = Builder.CreateAdd(Load, Inc->getStep());
}
+
auto *Store = Builder.CreateStore(Count, Addr);
if (isCounterPromotionEnabled())
PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
diff --git a/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll b/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll
index ebd510cfb5f08a..5a4ac3c3f67ce5 100644
--- a/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll
@@ -157,4 +157,5 @@ if.end: ; preds = %entry
return: ; preds = %if.end, %if.then
%1 = load i32, ptr %retval, align 4
ret i32 %1
-}
\ No newline at end of file
+}
+
>From 869a60b975600396793c81515e9d0f86e8b4e6e9 Mon Sep 17 00:00:00 2001
From: Alan Zhao <ayzhao at google.com>
Date: Thu, 19 Sep 2024 11:22:28 -0700
Subject: [PATCH 3/3] use add only per code review suggestion; -O3 lowers this
to a store i64 1 anyways
---
.../Instrumentation/InstrProfiling.cpp | 7 +--
.../conditional-counter-updates.ll | 54 ++++++++++---------
2 files changed, 30 insertions(+), 31 deletions(-)
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 6f05835aa4cfbc..8c7939dd519240 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -1265,19 +1265,14 @@ void InstrLowerer::lowerIncrement(InstrProfIncrementInst *Inc) {
Value *IncStep = Inc->getStep();
auto *CtrTy = IncStep->getType();
Value *Load = Builder.CreateLoad(CtrTy, Addr, "pgocount");
- Value *Count;
-
if (ConditionalCounterUpdate) {
Instruction *SplitBefore = Inc->getNextNode();
Value *Cmp = Builder.CreateIsNull(Load, "pgocount.ifzero");
Instruction *ThenBranch =
SplitBlockAndInsertIfThen(Cmp, SplitBefore, false);
Builder.SetInsertPoint(ThenBranch);
- Count = ConstantInt::get(CtrTy, 1);
- } else {
- Count = Builder.CreateAdd(Load, Inc->getStep());
}
-
+ auto *Count = Builder.CreateAdd(Load, Inc->getStep());
auto *Store = Builder.CreateStore(Count, Addr);
if (isCounterPromotionEnabled())
PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
diff --git a/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll b/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll
index 5a4ac3c3f67ce5..8d393c14ff4890 100644
--- a/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll
@@ -74,13 +74,14 @@ return: ; preds = %if.end, %if.then
; CHECK-LABEL: define void @fooint
; CHECK-NEXT: %pgocount = load i64, ptr @__profc_fooint, align 8
; CHECK-NEXT: %pgocount.ifzero = icmp eq i64 %pgocount, 0
-; CHECK-NEXT: br i1 %pgocount.ifzero, label %1, label %2
+; CHECK-NEXT: br i1 %pgocount.ifzero, label %1, label %3
; CHECK-LABEL: 1:
-; CHECK-NEXT: store i64 1, ptr @__profc_fooint, align 8
-; CHECK-NEXT: br label %2
+; CHECK-NEXT: %2 = add i64 %pgocount, 1
+; CHECK-NEXT: store i64 %2, ptr @__profc_fooint, align 8
+; CHECK-NEXT: br label %3
-; CHECK-LABEL: 2:
+; CHECK-LABEL: 3:
; CHECK-NEXT: ret void
define void @fooint() {
call void @llvm.instrprof.increment(ptr @__profn_fooint, i64 0, i32 1, i32 0)
@@ -94,46 +95,49 @@ define void @fooint() {
; CHECK-NEXT: store i32 %cond, ptr %cond.addr, align 4
; CHECK-NEXT: %pgocount = load i64, ptr @__profc_barint, align 8
; CHECK-NEXT: %pgocount.ifzero = icmp eq i64 %pgocount, 0
-; CHECK-NEXT: br i1 %pgocount.ifzero, label %0, label %1
+; CHECK-NEXT: br i1 %pgocount.ifzero, label %0, label %2
; CHECK-LABEL: 0: ; preds = %entry
-; CHECK-NEXT: store i64 1, ptr @__profc_barint, align 8
-; CHECK-NEXT: br label %1
+; CHECK-NEXT: %1 = add i64 %pgocount, 1
+; CHECK-NEXT: store i64 %1, ptr @__profc_barint, align 8
+; CHECK-NEXT: br label %2
-; CHECK-LABEL: 1: ; preds = %entry, %0
-; CHECK-NEXT: %2 = load i32, ptr %cond.addr, align 4
-; CHECK-NEXT: %cmp = icmp slt i32 %2, 0
+; CHECK-LABEL: 2: ; preds = %entry, %0
+; CHECK-NEXT: %3 = load i32, ptr %cond.addr, align 4
+; CHECK-NEXT: %cmp = icmp slt i32 %3, 0
; CHECK-NEXT: br i1 %cmp, label %if.then, label %if.end
-; CHECK-LABEL: if.then: ; preds = %1
+; CHECK-LABEL: if.then: ; preds = %2
; CHECK-NEXT: %pgocount1 = load i64, ptr getelementptr inbounds ([3 x i64], ptr @__profc_barint, i32 0, i32 1), align 8
; CHECK-NEXT: %pgocount.ifzero2 = icmp eq i64 %pgocount1, 0
-; CHECK-NEXT: br i1 %pgocount.ifzero2, label %3, label %4
+; CHECK-NEXT: br i1 %pgocount.ifzero2, label %4, label %6
-; CHECK-LABEL: 3: ; preds = %if.then
-; CHECK-NEXT: store i64 1, ptr getelementptr inbounds ([3 x i64], ptr @__profc_barint, i32 0, i32 1), align 8
-; CHECK-NEXT: br label %4
+; CHECK-LABEL: 4: ; preds = %if.then
+; CHECK-NEXT: %5 = add i64 %pgocount1, 1
+; CHECK-NEXT: store i64 %5, ptr getelementptr inbounds ([3 x i64], ptr @__profc_barint, i32 0, i32 1), align 8
+; CHECK-NEXT: br label %6
;
-; CHECK-LABEL: 4: ; preds = %if.then, %3
+; CHECK-LABEL: 6: ; preds = %if.then, %4
; CHECK-NEXT: store i32 -1, ptr %retval, align 4
; CHECK-NEXT: br label %return
;
-; CHECK-LABEL: if.end: ; preds = %1
+; CHECK-LABEL: if.end: ; preds = %2
; CHECK-NEXT: %pgocount3 = load i64, ptr getelementptr inbounds ([3 x i64], ptr @__profc_barint, i32 0, i32 2), align 8
; CHECK-NEXT: %pgocount.ifzero4 = icmp eq i64 %pgocount3, 0
-; CHECK-NEXT: br i1 %pgocount.ifzero4, label %5, label %6
+; CHECK-NEXT: br i1 %pgocount.ifzero4, label %7, label %9
;
-; CHECK-LABEL: 5: ; preds = %if.end
-; CHECK-NEXT: store i64 1, ptr getelementptr inbounds ([3 x i64], ptr @__profc_barint, i32 0, i32 2), align 8
-; CHECK-NEXT: br label %6
+; CHECK-LABEL: 7: ; preds = %if.end
+; CHECK-NEXT: %8 = add i64 %pgocount3, 1
+; CHECK-NEXT: store i64 %8, ptr getelementptr inbounds ([3 x i64], ptr @__profc_barint, i32 0, i32 2), align 8
+; CHECK-NEXT: br label %9
;
-; CHECK-LABEL: 6: ; preds = %if.end, %5
+; CHECK-LABEL: 9: ; preds = %if.end, %7
; CHECK-NEXT: store i32 0, ptr %retval, align 4
; CHECK-NEXT: br label %return
;
-; CHECK-LABEL: return: ; preds = %6, %4
-; CHECK-NEXT: %7 = load i32, ptr %retval, align 4
-; CHECK-NEXT: ret i32 %7
+; CHECK-LABEL: return: ; preds = %9, %6
+; CHECK-NEXT: %10 = load i32, ptr %retval, align 4
+; CHECK-NEXT: ret i32 %10
define i32 @barint(i32 %cond) #0 {
entry:
%retval = alloca i32, align 4
More information about the llvm-commits
mailing list