[llvm] f5b81aa - [InstrProf] Support conditional counter updates (#102542)

[via llvm-commits]

Author: gulfemsavrun
Date: 2024-08-16T12:33:50-07:00
New Revision: f5b81aa6ec371c13794aac82eb246a1a92966b43

URL: https://github.com/llvm/llvm-project/commit/f5b81aa6ec371c13794aac82eb246a1a92966b43
DIFF: https://github.com/llvm/llvm-project/commit/f5b81aa6ec371c13794aac82eb246a1a92966b43.diff

LOG: [InstrProf] Support conditional counter updates (#102542)

This patch adds support for conditional counter updates in single byte
counters mode to reduce the write contention by first checking whether
the counter is set before overwriting it.

---------

Co-authored-by: Juan Manuel Martinez Caamaño <jmartinezcaamao at gmail.com>

Added: 
    llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll

Modified: 
    llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp

Removed: 
    


################################################################################
diff  --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 1805ea89272ec7..1b3954a36699a0 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -132,6 +132,11 @@ cl::opt<bool> AtomicFirstCounter(
              "the entry counter)"),
     cl::init(false));
 
+cl::opt<bool> ConditionalCounterUpdate(
+    "conditional-counter-update",
+    cl::desc("Do conditional counter updates in single byte counters mode)"),
+    cl::init(false));
+
 // If the option is not specified, the default behavior about whether
 // counter promotion is done depends on how instrumentaiton lowering
 // pipeline is setup, i.e., the default value of true of this option
@@ -1213,6 +1218,17 @@ Value *InstrLowerer::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) {
 void InstrLowerer::lowerCover(InstrProfCoverInst *CoverInstruction) {
   auto *Addr = getCounterAddress(CoverInstruction);
   IRBuilder<> Builder(CoverInstruction);
+  if (ConditionalCounterUpdate) {
+    Instruction *SplitBefore = CoverInstruction->getNextNode();
+    auto &Ctx = CoverInstruction->getParent()->getContext();
+    auto *Int8Ty = llvm::Type::getInt8Ty(Ctx);
+    Value *Load = Builder.CreateLoad(Int8Ty, Addr, "pgocount");
+    Value *Cmp = Builder.CreateIsNotNull(Load, "pgocount.ifnonzero");
+    Instruction *ThenBranch =
+        SplitBlockAndInsertIfThen(Cmp, SplitBefore, false);
+    Builder.SetInsertPoint(ThenBranch);
+  }
+
   // We store zero to represent that this block is covered.
   Builder.CreateStore(Builder.getInt8(0), Addr);
   CoverInstruction->eraseFromParent();

diff  --git a/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll b/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll
new file mode 100644
index 00000000000000..998443ec0cd568
--- /dev/null
+++ b/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll
@@ -0,0 +1,69 @@
+; RUN: opt < %s -S -passes=instrprof -conditional-counter-update | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+ at __profn_foo = private constant [3 x i8] c"foo"
+ at __profn_bar = private constant [3 x i8] c"bar"
+
+; CHECK-LABEL: define void @foo
+; CHECK-NEXT: %pgocount = load i8, ptr @__profc_foo, align 1
+; CHECK-NEXT: %pgocount.ifnonzero = icmp ne i8 %pgocount, 0
+; CHECK-NEXT: br i1 %pgocount.ifnonzero, label %1, label %2
+
+; CHECK-LABEL: 1:
+; CHECK-NEXT: store i8 0, ptr @__profc_foo, align 1
+; CHECK-NEXT: br label %2
+
+; CHECK-LABEL: 2:
+; CHECK-NEXT: ret void
+define void @foo() {
+  call void @llvm.instrprof.cover(ptr @__profn_foo, i64 0, i32 1, i32 0)
+  ret void
+}
+
+; CHECK-LABEL: define i32 @bar
+; CHECK-LABEL: entry:
+; CHECK-NEXT: %retval = alloca i32, align 4
+; CHECK-NEXT: %cond.addr = alloca i32, align 4
+; CHECK-NEXT: store i32 %cond, ptr %cond.addr, align 4
+; CHECK-NEXT: %pgocount = load i8, ptr @__profc_bar, align 1
+; CHECK-NEXT: %pgocount.ifnonzero = icmp ne i8 %pgocount, 0
+; CHECK-NEXT: br i1 %pgocount.ifnonzero, label %0, label %1
+
+; CHECK-LABEL: 0:                                  ; preds = %entry
+; CHECK-NEXT: store i8 0, ptr @__profc_bar, align 1
+; CHECK-NEXT: br label %1
+
+; CHECK-LABEL: 1:                                  ; preds = %entry, %0
+; CHECK-NEXT: %2 = load i32, ptr %cond.addr, align 4
+; CHECK-NEXT: %cmp = icmp slt i32 %2, 0
+; CHECK-NEXT: br i1 %cmp, label %if.then, label %if.end
+
+; CHECK-LABEL: if.then:                            ; preds = %1
+; CHECK-NEXT: %pgocount1 = load i8, ptr getelementptr inbounds ([3 x i8], ptr @__profc_bar, i32 0, i32 1), align 1
+; CHECK-NEXT: %pgocount.ifnonzero2 = icmp ne i8 %pgocount1, 0
+; CHECK-NEXT: br i1 %pgocount.ifnonzero2, label %3, label %4
+define i32 @bar(i32 %cond) #0 {
+entry:
+  %retval = alloca i32, align 4
+  %cond.addr = alloca i32, align 4
+  store i32 %cond, ptr %cond.addr, align 4
+  call void @llvm.instrprof.cover(ptr @__profn_bar, i64 0, i32 3, i32 0)
+  %0 = load i32, ptr %cond.addr, align 4
+  %cmp = icmp slt i32 %0, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @llvm.instrprof.cover(ptr @__profn_bar, i64 0, i32 3, i32 1)
+  store i32 -1, ptr %retval, align 4
+  br label %return
+
+if.end:                                           ; preds = %entry
+  call void @llvm.instrprof.cover(ptr @__profn_bar, i64 0, i32 3, i32 2)
+  store i32 0, ptr %retval, align 4
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %1 = load i32, ptr %retval, align 4
+  ret i32 %1
+}