[llvm-branch-commits] [llvm] [PGO] Load branch uniformity profiles and mark availability (PR #221629)

Yaxun Liu via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Sun Nov 1 04:56:21 PST 2026


https://github.com/yxsamliu updated https://github.com/llvm/llvm-project/pull/221629

>From ea7c7ebd786baf11c5e0f7492f1098cf5c6e4585 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Wed, 28 Oct 2026 16:47:54 +0000
Subject: [PATCH] [PGO] Load branch uniformity profiles and mark availability

Uniformity profiles record full-wave block observations. Attach
uniformity.profile to functions with this data, and derive
branch.uniformity.profile from instrumented successor blocks with a
single conditional predecessor. Mark a branch only when every
instrumented outgoing edge used to classify it has a nonzero count
and a uniform classification. This summarizes successor observations;
it does not directly measure lane agreement at the branch.

Replace annotations covered by new observations, including removing
old positive hints. Leave existing annotations alone when uniformity
data is absent.

Add the hidden pgo-uniformity-metadata option, enabled by default.
Disabling it clears function, block and branch uniformity metadata
during profile use, including functions with no usable profile record,
while retaining ordinary profile counts and weights.
---
 .../Instrumentation/PGOInstrumentation.cpp    | 40 ++++++++-
 .../PGOProfile/uniformity-profile-use.ll      | 90 +++++++++++++++++++
 .../PGOInstrumentationTest.cpp                | 44 +++++++--
 3 files changed, 163 insertions(+), 11 deletions(-)
 create mode 100644 llvm/test/Transforms/PGOProfile/uniformity-profile-use.ll

diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 33a9a1049d527d..2a43b994c56223 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -160,6 +160,10 @@ STATISTIC(NumOfCSPGOMismatch,
 STATISTIC(NumOfCSPGOMissing, "Number of functions without profile in CSPGO.");
 STATISTIC(NumCoveredBlocks, "Number of basic blocks that were executed");
 
+static cl::opt<bool> PGOUniformityMetadata(
+    "pgo-uniformity-metadata", cl::init(true), cl::Hidden,
+    cl::desc("Enable uniformity profile metadata during PGO use"));
+
 namespace llvm {
 
 // Command line option to enable/disable the warning about a hash mismatch in
@@ -1620,9 +1624,9 @@ void PGOUseFunc::setBlockUniformityAttribute() {
   if (ProfileRecord.UniformityBits.empty())
     return;
 
-  // Annotate each uniform instrumented IR basic block so later codegen passes
-  // (MachineFunction) can consume it without relying on fragile block numbering
-  // heuristics.
+  // Mark the function as having uniformity profile, then annotate each uniform
+  // instrumented IR basic block so later codegen passes (MachineFunction) can
+  // consume it without relying on fragile block numbering heuristics.
   // Metadata presence on a terminator means uniform; divergent blocks have no
   // terminator metadata.
 
@@ -1631,19 +1635,36 @@ void PGOUseFunc::setBlockUniformityAttribute() {
 
   LLVMContext &Ctx = F.getContext();
   MDNode *UniformMD = MDNode::get(Ctx, {});
+  F.setMetadata(LLVMContext::MD_uniformity_profile, UniformMD);
+  DenseMap<CondBrInst *, bool> BranchUniformity;
   // A zero count has no uniformity observation even if its bitmap bit is set.
   auto HasUniformityObservation = [&](size_t I) {
     return I < ProfileRecord.Counts.size() && ProfileRecord.Counts[I] != 0;
   };
+
   for (size_t I = 0, E = InstrumentBBs.size(); I < E; ++I) {
     BasicBlock *BB = InstrumentBBs[I];
     if (!BB)
       continue;
     bool IsUniform =
         HasUniformityObservation(I) && ProfileRecord.isBlockUniform(I);
+    // A counter placed in a block with a single conditional predecessor also
+    // measures the active lanes on that outgoing edge. Record the branch as
+    // uniform only when every instrumented outgoing edge is uniform.
+    if (BasicBlock *Pred = BB->getSinglePredecessor()) {
+      if (auto *Branch = dyn_cast<CondBrInst>(Pred->getTerminator())) {
+        auto It = BranchUniformity.try_emplace(Branch, true).first;
+        It->second &= IsUniform;
+      }
+    }
+
     BB->getTerminator()->setMetadata(LLVMContext::MD_block_uniformity_profile,
                                      IsUniform ? UniformMD : nullptr);
   }
+  for (auto [Branch, IsUniform] : BranchUniformity) {
+    Branch->setMetadata(LLVMContext::MD_branch_uniformity_profile,
+                        IsUniform ? UniformMD : nullptr);
+  }
 
   LLVM_DEBUG({
     dbgs() << "PGO: Set block uniformity profile for " << F.getName() << ": ";
@@ -2114,6 +2135,16 @@ static bool annotateAllFunctions(
 
   bool HasSingleByteCoverage = PGOReader->hasSingleByteCoverage();
   for (auto &F : M) {
+    if (!PGOUniformityMetadata) {
+      // Also remove existing annotations when the replacement profile has no
+      // usable record for this function.
+      F.setMetadata(LLVMContext::MD_uniformity_profile, nullptr);
+      for (BasicBlock &BB : F) {
+        Instruction *TI = BB.getTerminator();
+        TI->setMetadata(LLVMContext::MD_block_uniformity_profile, nullptr);
+        TI->setMetadata(LLVMContext::MD_branch_uniformity_profile, nullptr);
+      }
+    }
     if (skipPGOUse(Opts, F))
       continue;
     TargetLibraryInfo &TLI = LookupTLI(F);
@@ -2162,7 +2193,8 @@ static bool annotateAllFunctions(
     Func.setBranchWeights();
     Func.annotateValueSites();
     Func.annotateIrrLoopHeaderWeights();
-    Func.setBlockUniformityAttribute();
+    if (PGOUniformityMetadata)
+      Func.setBlockUniformityAttribute();
     PGOUseFunc::FuncFreqAttr FreqAttr = Func.getFuncFreqAttr();
     if (FreqAttr == PGOUseFunc::FFA_Cold)
       ColdFunctions.push_back(&F);
diff --git a/llvm/test/Transforms/PGOProfile/uniformity-profile-use.ll b/llvm/test/Transforms/PGOProfile/uniformity-profile-use.ll
new file mode 100644
index 00000000000000..0621c6ba57de62
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/uniformity-profile-use.ll
@@ -0,0 +1,90 @@
+; RUN: split-file %s %t
+; RUN: opt -passes=pgo-instr-gen -pgo-instrument-entry -S %t/input.ll | FileCheck %s --check-prefix=GEN
+; RUN: %python %t/raw.py > %t/profile.raw
+; RUN: llvm-profdata merge %t/profile.raw -o %t/profile
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/profile -S %t/input.ll -o %t/default.ll
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/profile -pgo-uniformity-metadata=true -S %t/input.ll -o %t/enabled.ll
+; RUN: diff %t/default.ll %t/enabled.ll
+; RUN: FileCheck %s --check-prefixes=ON,COUNTS < %t/default.ll
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/profile -pgo-uniformity-metadata=false -S %t/input.ll | FileCheck %s --check-prefix=COUNTS --implicit-check-not=uniformity.profile
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/profile -pgo-uniformity-metadata=false -S %t/default.ll | FileCheck %s --check-prefix=COUNTS --implicit-check-not=uniformity.profile
+; RUN: %python %t/raw.py missing > %t/missing.raw
+; RUN: llvm-profdata merge %t/missing.raw -o %t/missing.profile
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/missing.profile -pgo-uniformity-metadata=false -S %t/default.ll | FileCheck %s --check-prefix=COUNTS --implicit-check-not=uniformity.profile
+; RUN: %python %t/raw.py mismatch > %t/mismatch.raw
+; RUN: llvm-profdata merge %t/mismatch.raw -o %t/mismatch.profile
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/mismatch.profile -pgo-uniformity-metadata=false -S %t/default.ll | FileCheck %s --check-prefix=COUNTS --implicit-check-not=uniformity.profile
+; RUN: %python %t/raw.py zero > %t/zero.raw
+; RUN: llvm-profdata merge %t/zero.raw -o %t/zero.profile
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/zero.profile -pgo-uniformity-metadata=false -S %t/default.ll | FileCheck %s --check-prefix=ZERO --implicit-check-not=uniformity.profile
+
+; The fixture follows the real instrumented counter order. Both block counters
+; observe full-wave entries. The select counter is unrelated to block metadata.
+; GEN-LABEL: define void @diamond
+; GEN: entry:
+; GEN: call void @llvm.instrprof.increment({{.*}}i64 [[HASH:942389667449461396]], i32 3, i32 0)
+; GEN: b:
+; GEN: call void @llvm.instrprof.increment({{.*}}i64 [[HASH]], i32 3, i32 1)
+; GEN: call void @llvm.instrprof.increment.step({{.*}}i64 [[HASH]], i32 3, i32 2,
+
+; Disabling uniformity preserves ordinary entry, branch and select counts.
+; It also clears existing uniformity annotations before missing, mismatched or zero-count records
+; can skip normal annotation. No consumer-specific switch is required.
+; ON: define void @diamond({{.*}}!uniformity.profile
+; ON: br i1 %cond, {{.*}}!block.uniformity.profile{{.*}}!branch.uniformity.profile
+; ON: b:
+; ON: br label %exit, {{.*}}!block.uniformity.profile
+; COUNTS-DAG: !{!"function_entry_count", i64 6400}
+; COUNTS-DAG: !{!"branch_weights", i32 3200, i32 3200}
+; COUNTS-DAG: !{!"branch_weights", i32 1600, i32 4800}
+; ZERO: !{!"function_entry_count", i64 0}
+
+;--- input.ll
+source_filename = "uniformity-profile-use.ll"
+target triple = "amdgcn-amd-amdhsa"
+define void @diamond(i1 %cond, i1 %select_cond, ptr %p) {
+entry:
+  br i1 %cond, label %a, label %b
+a:
+  store volatile i32 1, ptr %p
+  br label %exit
+b:
+  store volatile i32 2, ptr %p
+  br label %exit
+exit:
+  %value = select i1 %select_cond, i32 1, i32 2
+  store volatile i32 %value, ptr %p
+  ret void
+}
+
+;--- raw.py
+import hashlib
+import struct
+import sys
+
+mode = sys.argv[1] if len(sys.argv) > 1 else "uniform"
+name = b"missing" if mode == "missing" else b"diamond"
+name_ref = int.from_bytes(hashlib.md5(name).digest()[:8], "little")
+# Raw v11, IR entry-first layout; hash checked against pgo-instr-gen above.
+func_hash = 942389667449461396
+if mode == "mismatch":
+    func_hash += 1
+lanes = [6400, 3200, 1600]
+if mode == "zero":
+    lanes = [0, 0, 0]
+uniform = lanes[:]
+version = 11 | (1 << 56) | (1 << 58)
+num_counters = len(lanes)
+counter_delta = 72
+uniform_delta = counter_delta + num_counters * 8
+names_delta = uniform_delta + len(uniform) * 8
+names = bytes([len(name), 0]) + name
+header = [0xff6c70726f667281, version, 0, 1, 0, num_counters, 0,
+          0, 0, len(uniform), 0, uniform_delta, len(names), counter_delta,
+          uniform_delta, names_delta, 0, 0, 2]
+record = struct.pack("<7QI4HI", name_ref, func_hash, counter_delta,
+                     uniform_delta, 0, 0, 0, num_counters, 0, 0, 0, 64, 0)
+counts = lanes + uniform
+sys.stdout.buffer.write(struct.pack("<19Q", *header) + record +
+                        struct.pack("<" + "Q" * len(counts), *counts) +
+                        names + bytes((-len(names)) % 8))
diff --git a/llvm/unittests/Transforms/Instrumentation/PGOInstrumentationTest.cpp b/llvm/unittests/Transforms/Instrumentation/PGOInstrumentationTest.cpp
index 9542955dcb7235..4da577871ac6ea 100644
--- a/llvm/unittests/Transforms/Instrumentation/PGOInstrumentationTest.cpp
+++ b/llvm/unittests/Transforms/Instrumentation/PGOInstrumentationTest.cpp
@@ -27,6 +27,7 @@ namespace {
 using namespace llvm;
 
 using testing::_;
+using ::testing::Combine;
 using ::testing::DoDefault;
 using ::testing::Invoke;
 using ::testing::NotNull;
@@ -192,12 +193,15 @@ TEST_P(PGOInstrumentationGenTest, Instrumented) {
   EXPECT_FALSE(IRInstrVar->isDeclaration());
 }
 
-struct PGOInstrumentationUseTest : Test, WithParamInterface<bool> {};
+struct PGOInstrumentationUseTest : Test,
+                                   WithParamInterface<std::tuple<bool, bool>> {
+};
 
-INSTANTIATE_TEST_SUITE_P(ExistingMetadata, PGOInstrumentationUseTest,
-                         Values(false, true));
+INSTANTIATE_TEST_SUITE_P(ProfileAvailability, PGOInstrumentationUseTest,
+                         Combine(Values(false, true), Values(false, true)));
 
-TEST_P(PGOInstrumentationUseTest, BlockUniformityMetadataUsesPresence) {
+TEST_P(PGOInstrumentationUseTest, UniformityMetadataUsesPresence) {
+  const auto [HasUniformityProfile, HasExistingMetadata] = GetParam();
   static constexpr StringRef Code = R"(
     define i32 @f(i1 %cond) {
     entry:
@@ -264,7 +268,8 @@ TEST_P(PGOInstrumentationUseTest, BlockUniformityMetadataUsesPresence) {
     if (ZeroCountCase != 0)
       Counts[ZeroCountCase - 1] = 0;
     NamedInstrProfRecord Record(ProfileName, FunctionHash, Counts);
-    Record.UniformityBits = {static_cast<uint8_t>(UniformityMask)};
+    if (HasUniformityProfile)
+      Record.UniformityBits = {static_cast<uint8_t>(UniformityMask)};
 
     InstrProfWriter Writer;
     ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::IRInstrumentation),
@@ -281,17 +286,40 @@ TEST_P(PGOInstrumentationUseTest, BlockUniformityMetadataUsesPresence) {
     ASSERT_THAT(UseModule, NotNull());
     Function *UseFunction = UseModule->getFunction("f");
     ASSERT_THAT(UseFunction, NotNull());
-    if (GetParam()) {
+    if (HasExistingMetadata) {
       MDNode *UniformMD = MDNode::get(Context, {});
+      UseFunction->setMetadata(LLVMContext::MD_uniformity_profile, UniformMD);
       for (BasicBlock &BB : *UseFunction)
         BB.getTerminator()->setMetadata(
             LLVMContext::MD_block_uniformity_profile, UniformMD);
+      UseFunction->getEntryBlock().getTerminator()->setMetadata(
+          LLVMContext::MD_branch_uniformity_profile, UniformMD);
     }
     ModulePassManager UseMPM;
     UseMPM.addPass(PGOInstrumentationUse("/profile.profdata", "", false, FS));
     UseMPM.run(*UseModule, MAM);
     EXPECT_FALSE(verifyModule(*UseModule, &errs()));
 
+    MDNode *FunctionMD =
+        UseFunction->getMetadata(LLVMContext::MD_uniformity_profile);
+    // Missing uniformity data preserves existing hints. Observed nonuniformity
+    // replaces positive hints covered by the new profile.
+    EXPECT_EQ(FunctionMD != nullptr,
+              HasUniformityProfile || HasExistingMetadata);
+    if (FunctionMD)
+      EXPECT_EQ(FunctionMD->getNumOperands(), 0u);
+
+    auto *Branch =
+        cast<CondBrInst>(UseFunction->getEntryBlock().getTerminator());
+    MDNode *BranchMD =
+        Branch->getMetadata(LLVMContext::MD_branch_uniformity_profile);
+    EXPECT_EQ(BranchMD != nullptr,
+              HasUniformityProfile
+                  ? UniformityMask == 3 && Counts[0] != 0 && Counts[1] != 0
+                  : HasExistingMetadata);
+    if (BranchMD)
+      EXPECT_EQ(BranchMD->getNumOperands(), 0u);
+
     for (unsigned I = 0; I < NumCounters; ++I) {
       BasicBlock *BB = nullptr;
       for (BasicBlock &Candidate : *UseFunction)
@@ -301,7 +329,9 @@ TEST_P(PGOInstrumentationUseTest, BlockUniformityMetadataUsesPresence) {
 
       MDNode *MD = BB->getTerminator()->getMetadata(
           LLVMContext::MD_block_uniformity_profile);
-      bool IsUniform = (UniformityMask & (1u << I)) != 0 && Counts[I] != 0;
+      bool IsUniform = HasUniformityProfile
+                           ? (UniformityMask & (1u << I)) != 0 && Counts[I] != 0
+                           : HasExistingMetadata;
       EXPECT_EQ(MD != nullptr, IsUniform);
       if (MD)
         EXPECT_EQ(MD->getNumOperands(), 0u);



More information about the llvm-branch-commits mailing list