[llvm] [InstrProf] Support conditional counter updates (PR #102542)

[via llvm-commits]

https://github.com/gulfemsavrun updated https://github.com/llvm/llvm-project/pull/102542

>From f915b0cd905d0bef9383052a916a978578dd7ca0 Mon Sep 17 00:00:00 2001
From: Gulfem Savrun Yeniceri <gulfem at google.com>
Date: Thu, 8 Aug 2024 21:36:55 +0000
Subject: [PATCH] [InstrProf] Support conditional counter updates

This patch adds support of conditional counter updates
in single byte counters mode to reduce the write contention
by first checking whether the counter is set before
overwriting it.
---
 .../Instrumentation/InstrProfiling.cpp        | 17 +++++
 .../conditional-counter-updates.ll            | 69 +++++++++++++++++++
 2 files changed, 86 insertions(+)
 create mode 100644 llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll

diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 1805ea89272ec7..10659334107028 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -132,6 +132,11 @@ cl::opt<bool> AtomicFirstCounter(
              "the entry counter)"),
     cl::init(false));
 
+cl::opt<bool> ConditionalCounterUpdate(
+    "conditional-counter-update",
+    cl::desc("Do conditional counter updates in single byte counters mode)"),
+    cl::init(false));
+
 // If the option is not specified, the default behavior about whether
 // counter promotion is done depends on how instrumentaiton lowering
 // pipeline is setup, i.e., the default value of true of this option
@@ -1213,6 +1218,18 @@ Value *InstrLowerer::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) {
 void InstrLowerer::lowerCover(InstrProfCoverInst *CoverInstruction) {
   auto *Addr = getCounterAddress(CoverInstruction);
   IRBuilder<> Builder(CoverInstruction);
+  if (ConditionalCounterUpdate) {
+    Instruction *SplitBefore = CoverInstruction->getNextNode();
+    auto &Ctx = CoverInstruction->getParent()->getContext();
+    auto *Int8Ty = llvm::Type::getInt8Ty(Ctx);
+    Value *Load = Builder.CreateLoad(Int8Ty, Addr, "pgocount");
+    Value *Cmp = Builder.CreateICmpNE(Load, ConstantInt::get(Int8Ty, 0),
+                                      "pgocount.ifnonzero");
+    Instruction *ThenBranch =
+        SplitBlockAndInsertIfThen(Cmp, SplitBefore, false);
+    Builder.SetInsertPoint(ThenBranch);
+  }
+
   // We store zero to represent that this block is covered.
   Builder.CreateStore(Builder.getInt8(0), Addr);
   CoverInstruction->eraseFromParent();
diff --git a/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll b/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll
new file mode 100644
index 00000000000000..998443ec0cd568
--- /dev/null
+++ b/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll
@@ -0,0 +1,69 @@
+; RUN: opt < %s -S -passes=instrprof -conditional-counter-update | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+ at __profn_foo = private constant [3 x i8] c"foo"
+ at __profn_bar = private constant [3 x i8] c"bar"
+
+; CHECK-LABEL: define void @foo
+; CHECK-NEXT: %pgocount = load i8, ptr @__profc_foo, align 1
+; CHECK-NEXT: %pgocount.ifnonzero = icmp ne i8 %pgocount, 0
+; CHECK-NEXT: br i1 %pgocount.ifnonzero, label %1, label %2
+
+; CHECK-LABEL: 1:
+; CHECK-NEXT: store i8 0, ptr @__profc_foo, align 1
+; CHECK-NEXT: br label %2
+
+; CHECK-LABEL: 2:
+; CHECK-NEXT: ret void
+define void @foo() {
+  call void @llvm.instrprof.cover(ptr @__profn_foo, i64 0, i32 1, i32 0)
+  ret void
+}
+
+; CHECK-LABEL: define i32 @bar
+; CHECK-LABEL: entry:
+; CHECK-NEXT: %retval = alloca i32, align 4
+; CHECK-NEXT: %cond.addr = alloca i32, align 4
+; CHECK-NEXT: store i32 %cond, ptr %cond.addr, align 4
+; CHECK-NEXT: %pgocount = load i8, ptr @__profc_bar, align 1
+; CHECK-NEXT: %pgocount.ifnonzero = icmp ne i8 %pgocount, 0
+; CHECK-NEXT: br i1 %pgocount.ifnonzero, label %0, label %1
+
+; CHECK-LABEL: 0:                                  ; preds = %entry
+; CHECK-NEXT: store i8 0, ptr @__profc_bar, align 1
+; CHECK-NEXT: br label %1
+
+; CHECK-LABEL: 1:                                  ; preds = %entry, %0
+; CHECK-NEXT: %2 = load i32, ptr %cond.addr, align 4
+; CHECK-NEXT: %cmp = icmp slt i32 %2, 0
+; CHECK-NEXT: br i1 %cmp, label %if.then, label %if.end
+
+; CHECK-LABEL: if.then:                            ; preds = %1
+; CHECK-NEXT: %pgocount1 = load i8, ptr getelementptr inbounds ([3 x i8], ptr @__profc_bar, i32 0, i32 1), align 1
+; CHECK-NEXT: %pgocount.ifnonzero2 = icmp ne i8 %pgocount1, 0
+; CHECK-NEXT: br i1 %pgocount.ifnonzero2, label %3, label %4
+define i32 @bar(i32 %cond) #0 {
+entry:
+  %retval = alloca i32, align 4
+  %cond.addr = alloca i32, align 4
+  store i32 %cond, ptr %cond.addr, align 4
+  call void @llvm.instrprof.cover(ptr @__profn_bar, i64 0, i32 3, i32 0)
+  %0 = load i32, ptr %cond.addr, align 4
+  %cmp = icmp slt i32 %0, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @llvm.instrprof.cover(ptr @__profn_bar, i64 0, i32 3, i32 1)
+  store i32 -1, ptr %retval, align 4
+  br label %return
+
+if.end:                                           ; preds = %entry
+  call void @llvm.instrprof.cover(ptr @__profn_bar, i64 0, i32 3, i32 2)
+  store i32 0, ptr %retval, align 4
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %1 = load i32, ptr %retval, align 4
+  ret i32 %1
+}