[llvm] InstrProf: Mark BiasLI as invariant. (PR #95588)

[NAKAMURA Takumi via llvm-commits]

https://github.com/chapuni created https://github.com/llvm/llvm-project/pull/95588

Bias doesn't change after startup.

The test is enhanced for optimized sequences and atomic ops.

>From 5248130c45e6b5dd4c8349d507327b3cfbde3104 Mon Sep 17 00:00:00 2001
From: NAKAMURA Takumi <geek4civic at gmail.com>
Date: Sat, 15 Jun 2024 03:16:04 +0900
Subject: [PATCH] InstrProf: Mark BiasLI as invariant.

Bias doesn't change after startup.
---
 .../Instrumentation/InstrProfiling.cpp        |  3 ++
 .../runtime-counter-relocation.ll             | 42 +++++++++++++++----
 .../PGOProfile/counter_promo_with_bias.ll     |  2 +-
 3 files changed, 37 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 0c79eaa812b5f..8ea76f89b7dd2 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -927,6 +927,9 @@ Value *InstrLowerer::getCounterAddress(InstrProfCntrInstBase *I) {
         Bias->setComdat(M.getOrInsertComdat(Bias->getName()));
     }
     BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias);
+    // Bias doesn't change after startup.
+    BiasLI->setMetadata(LLVMContext::MD_invariant_load,
+                        MDNode::get(M.getContext(), std::nullopt));
   }
   auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), BiasLI);
   return Builder.CreateIntToPtr(Add, Addr->getType());
diff --git a/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll b/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll
index 44a2efd4a959a..3f0dbef387d2f 100644
--- a/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll
@@ -1,25 +1,49 @@
 ; RUN: opt < %s -S -passes=instrprof | FileCheck %s
 ; RUN: opt < %s -S -passes=instrprof -runtime-counter-relocation | FileCheck -check-prefixes=RELOC %s
+; RUN: opt < %s -S -passes=instrprof,inline,gvn -runtime-counter-relocation | FileCheck -check-prefixes=RELOC,RELOCOPT %s
+; RUN: opt < %s -S -passes=instrprof            -runtime-counter-relocation -instrprof-atomic-counter-update-all | FileCheck -check-prefixes=ATOMIC %s
+; RUN: opt < %s -S -passes=instrprof,inline,gvn -runtime-counter-relocation -instrprof-atomic-counter-update-all | FileCheck -check-prefixes=ATOMIC,ATOMICOPT %s
 
 target triple = "x86_64-unknown-linux-gnu"
 
 @__profn_foo = private constant [3 x i8] c"foo"
+ at __profn_bar = private constant [3 x i8] c"bar"
 ; RELOC: $__llvm_profile_counter_bias = comdat any
 ; RELOC: @__llvm_profile_counter_bias = linkonce_odr hidden global i64 0, comdat
 
 ; CHECK-LABEL: define void @foo
-; CHECK-NEXT: %pgocount = load i64, ptr @__profc_foo
-; CHECK-NEXT: %1 = add i64 %pgocount, 1
-; CHECK-NEXT: store i64 %1, ptr @__profc_foo
+; CHECK-NEXT: %[[PGOCOUNT:.+]] = load i64, ptr @__profc_foo
+; CHECK-NEXT: %[[PGOCOUNTINC:.+]] = add i64 %[[PGOCOUNT]], 1
+; CHECK-NEXT: store i64 %[[PGOCOUNTINC]], ptr @__profc_foo
 ; RELOC-LABEL: define void @foo
-; RELOC-NEXT: %1 = load i64, ptr @__llvm_profile_counter_bias
-; RELOC-NEXT: %2 = add i64 ptrtoint (ptr @__profc_foo to i64), %1
-; RELOC-NEXT: %3 = inttoptr i64 %2 to ptr
-; RELOC-NEXT: %pgocount = load i64, ptr %3
-; RELOC-NEXT: %4 = add i64 %pgocount, 1
-; RELOC-NEXT: store i64 %4, ptr %3
+; RELOC-NEXT: %[[BIAS:.+]] = load i64, ptr @__llvm_profile_counter_bias, align 8, !invariant.load !0
+; RELOC-NEXT: %[[PROFC_BIAS:.+]] = add i64 ptrtoint (ptr @__profc_foo to i64), %[[BIAS]]
+; RELOC-NEXT: %[[PROFC_ADDR:.+]] = inttoptr i64 %[[PROFC_BIAS]] to ptr
+; RELOC-NEXT: %[[PGOCOUNT:.+]] = load i64, ptr %[[PROFC_ADDR]]
+; RELOC-NEXT: %[[PGOCOUNTINC:.+]] = add i64 %[[PGOCOUNT]], 1
+; RELOC-NEXT: store i64 %[[PGOCOUNTINC]], ptr %[[PROFC_ADDR]]
+; RELOCOPT-NEXT: %[[PROFC_BIAS1:.+]] = add i64 ptrtoint (ptr @__profc_bar to i64), %[[BIAS]]
+; RELOCOPT-NEXT: %[[PROFC_ADDR1:.+]] = inttoptr i64 %[[PROFC_BIAS1]] to ptr
+; RELOCOPT-NEXT: %[[PGOCOUNT1:.+]] = load i64, ptr %[[PROFC_ADDR1]]
+; RELOCOPT-NEXT: %[[PGOCOUNTINC1:.+]] = add i64 %[[PGOCOUNT1]], 1
+; RELOCOPT-NEXT: store i64 %[[PGOCOUNTINC1]], ptr %[[PROFC_ADDR1]]
+; ATOMIC-LABEL: define void @foo
+; ATOMIC-NEXT: %[[BIAS:.+]] = load i64, ptr @__llvm_profile_counter_bias, align 8, !invariant.load !0
+; ATOMIC-NEXT: %[[PROFC_BIAS:.+]] = add i64 ptrtoint (ptr @__profc_foo to i64), %[[BIAS]]
+; ATOMIC-NEXT: %[[PROFC_ADDR:.+]] = inttoptr i64 %[[PROFC_BIAS]] to ptr
+; ATOMIC-NEXT: %[[PGOCOUNTINC:.+]] = atomicrmw add ptr %[[PROFC_ADDR]], i64 1 monotonic
+; ATOMICOPT-NEXT: %[[PROFC_BIAS1:.+]] = add i64 ptrtoint (ptr @__profc_bar to i64), %[[BIAS]]
+; ATOMICOPT-NEXT: %[[PROFC_ADDR1:.+]] = inttoptr i64 %[[PROFC_BIAS1]] to ptr
+; ATOMICOPT-NEXT: %[[PGOCOUNTINC1:.+]] = atomicrmw add ptr %[[PROFC_ADDR1]], i64 1 monotonic
+
+define void @bar() {
+  call void @llvm.instrprof.increment(ptr @__profn_bar, i64 0, i32 1, i32 0)
+  ret void
+}
+
 define void @foo() {
   call void @llvm.instrprof.increment(ptr @__profn_foo, i64 0, i32 1, i32 0)
+  call void @bar()
   ret void
 }
 
diff --git a/llvm/test/Transforms/PGOProfile/counter_promo_with_bias.ll b/llvm/test/Transforms/PGOProfile/counter_promo_with_bias.ll
index c489a9f8eea84..7b1da764e776e 100644
--- a/llvm/test/Transforms/PGOProfile/counter_promo_with_bias.ll
+++ b/llvm/test/Transforms/PGOProfile/counter_promo_with_bias.ll
@@ -6,7 +6,7 @@ target triple = "x86_64-unknown-linux-gnu"
 
 define void @foo(i1 %c) {
 entry:
-; CHECK: %[[BIAS:[0-9]+]] = load i64, ptr @__llvm_profile_counter_bias
+; CHECK: %[[BIAS:.+]] = load i64, ptr @__llvm_profile_counter_bias
   br label %while.cond
 
 while.cond:                                       ; preds = %land.rhs, %while.cond.preheader