[llvm-branch-commits] [llvm] [PGO] Load branch uniformity profiles and mark availability (PR #221629)
Yaxun Liu via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Sep 11 06:52:29 PDT 2026
https://github.com/yxsamliu updated https://github.com/llvm/llvm-project/pull/221629
>From 06dd54cdaa44fed2742a513aae5d5de5bfabf3d6 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Fri, 11 Sep 2026 09:34:15 -0400
Subject: [PATCH 1/2] [IR] Align uniformity metadata with RFC v2
---
llvm/docs/LangRef.md | 55 ++++++++++++-------
llvm/include/llvm/IR/FixedMetadataKinds.def | 1 +
llvm/lib/IR/Verifier.cpp | 19 ++++---
.../block-uniformity-profile-metadata.ll | 4 +-
.../test/Verifier/block-uniformity-profile.ll | 15 ++---
llvm/test/Verifier/uniformity-profile.ll | 27 +++++++++
6 files changed, 81 insertions(+), 40 deletions(-)
create mode 100644 llvm/test/Verifier/uniformity-profile.ll
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index ba562c88a7c22..b66146e9b3a8a 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -7897,21 +7897,38 @@ section is not marked as readable or writable and it uses the section flag
!0 = !{}
```
+(md_uniformity_profile)=
+
+#### '`uniformity.profile`' Metadata
+
+`uniformity.profile` metadata indicates that uniformity profile data was
+loaded for a function. It may be attached only to a function definition and
+must be an empty metadata node.
+
+This marker lets an optimization distinguish a function without uniformity
+profile data from one whose individual blocks or branches are not known to be
+uniform. Missing block or branch metadata may mean that the control flow was
+divergent, was not observed, or lacked enough profile information.
+
+```llvm
+define void @example() !uniformity.profile !0 {
+ ret void
+}
+
+!0 = !{}
+```
+
(md_block_uniformity_profile)=
#### '`block.uniformity.profile`' Metadata
`block.uniformity.profile` metadata records observed SIMT execution uniformity
-from an instrumentation profile. It may be attached to a function definition
-or a terminator instruction and must be an empty metadata node.
-
-On a function definition, this metadata indicates that a uniformity profile
-was loaded. On a terminator, it indicates that the profile classified the
-basic block as usually executing with all lanes active. This classification
-may tolerate some divergent executions in the profile. A function attachment
-without any annotated terminators can represent a profile in which no instrumented block was
-uniform. An unannotated block is not known to be uniform; it may be divergent
-or lack a uniformity observation.
+from an instrumentation profile. It may be attached only to a terminator
+instruction and must be an empty metadata node. It indicates that the profile
+classified the containing basic block as usually executing with all lanes
+active. This classification may tolerate some divergent executions in the
+profile. An unannotated block is not known to be uniform; it may be divergent,
+unobserved, or lack enough profile information.
Block uniformity does not describe the uniformity of a branch decision. For
example, all lanes may reach a block and then take different successors. See
@@ -7923,9 +7940,9 @@ guarantee uniform execution on other inputs and must not be used to justify
transformations that require uniformity for correctness.
```llvm
-define void @example(i1 %condition) !block.uniformity.profile !0 {
+define void @example(i1 %condition) !uniformity.profile !0 {
entry:
- br i1 %condition, label %then, label %else, !block.uniformity.profile !0
+ br i1 %condition, label %then, label %else, !block.uniformity.profile !0, !branch.uniformity.profile !0
then:
ret void
else:
@@ -7939,15 +7956,15 @@ else:
#### '`branch.uniformity.profile`' Metadata
-`branch.uniformity.profile` metadata records that a profile classified a
-conditional branch as usually uniform across lanes in a SIMT execution group.
-It may be attached only to a conditional branch instruction and must be an
-empty metadata node. A uniform decision means the lanes choose the same
-successor on that execution; the chosen successor can vary between executions.
-The profile classification may tolerate some divergent executions.
+`branch.uniformity.profile` metadata records a profile-derived hint that every
+instrumented successor used to classify a conditional branch usually executed
+with all lanes active. It may be attached only to a conditional branch
+instruction and must be an empty metadata node. The profiler does not directly
+observe every branch decision or whether only the currently active lanes chose
+the same successor.
An unannotated branch is not known to be uniform. Optimizations can use the
-function-level {ref}`block.uniformity.profile <md_block_uniformity_profile>`
+function-level {ref}`uniformity.profile <md_uniformity_profile>`
attachment to distinguish a function with uniformity profile information from
one without it.
diff --git a/llvm/include/llvm/IR/FixedMetadataKinds.def b/llvm/include/llvm/IR/FixedMetadataKinds.def
index 76e862eae3aee..54fbc1a3065be 100644
--- a/llvm/include/llvm/IR/FixedMetadataKinds.def
+++ b/llvm/include/llvm/IR/FixedMetadataKinds.def
@@ -69,3 +69,4 @@ LLVM_FIXED_MD_KIND(MD_callgraph, "callgraph", 54)
LLVM_FIXED_MD_KIND(MD_metadata_section_kind, "metadata_section_kind", 55)
LLVM_FIXED_MD_KIND(MD_branch_uniformity_profile, "branch.uniformity.profile",
56)
+LLVM_FIXED_MD_KIND(MD_uniformity_profile, "uniformity.profile", 57)
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 97560b4af30cb..751d2b1c941db 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -586,11 +586,11 @@ void Verifier::visitGlobalValue(const GlobalValue &GV) {
Check(!GO->hasMetadata(LLVMContext::MD_branch_uniformity_profile),
"branch.uniformity.profile is only valid on conditional branches",
GO);
- if (GO->hasMetadata(LLVMContext::MD_block_uniformity_profile))
+ Check(!GO->hasMetadata(LLVMContext::MD_block_uniformity_profile),
+ "block.uniformity.profile is only valid on terminators", GO);
+ if (GO->hasMetadata(LLVMContext::MD_uniformity_profile))
Check(isa<Function>(GO) && !GO->isDeclaration(),
- "block.uniformity.profile is only valid on function definitions "
- "and terminators",
- GO);
+ "uniformity.profile is only valid on function definitions", GO);
if (const MDNode *Associated =
GO->getMetadata(LLVMContext::MD_associated)) {
@@ -2769,9 +2769,9 @@ void Verifier::verifyUnknownProfileMetadata(MDNode *MD) {
void Verifier::verifyFunctionMetadata(
ArrayRef<std::pair<unsigned, MDNode *>> MDs) {
for (const auto &Pair : MDs) {
- if (Pair.first == LLVMContext::MD_block_uniformity_profile)
+ if (Pair.first == LLVMContext::MD_uniformity_profile)
Check(Pair.second->getNumOperands() == 0,
- "block.uniformity.profile must be an empty node", Pair.second);
+ "uniformity.profile must be an empty node", Pair.second);
if (Pair.first == LLVMContext::MD_prof) {
MDNode *MD = Pair.second;
Check(MD->getNumOperands() >= 2,
@@ -6061,13 +6061,14 @@ void Verifier::visitInstruction(Instruction &I) {
if (MDNode *MD = I.getMetadata(LLVMContext::MD_block_uniformity_profile)) {
Check(I.isTerminator(),
- "block.uniformity.profile is only valid on function definitions "
- "and terminators",
- &I);
+ "block.uniformity.profile is only valid on terminators", &I);
Check(MD->getNumOperands() == 0,
"block.uniformity.profile must be an empty node", &I, MD);
}
+ Check(!I.getMetadata(LLVMContext::MD_uniformity_profile),
+ "uniformity.profile is only valid on function definitions", &I);
+
if (MDNode *MD = I.getMetadata(LLVMContext::MD_branch_uniformity_profile)) {
Check(isa<CondBrInst>(I),
"branch.uniformity.profile is only valid on conditional branches",
diff --git a/llvm/test/Bitcode/block-uniformity-profile-metadata.ll b/llvm/test/Bitcode/block-uniformity-profile-metadata.ll
index a98f1cebda69d..bcdd48bf2b505 100644
--- a/llvm/test/Bitcode/block-uniformity-profile-metadata.ll
+++ b/llvm/test/Bitcode/block-uniformity-profile-metadata.ll
@@ -1,8 +1,8 @@
; RUN: llvm-as < %s | llvm-dis | FileCheck %s
; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
-define void @branch_metadata(i1 %cond) !block.uniformity.profile !0 {
-; CHECK-LABEL: define void @branch_metadata(i1 %cond) !block.uniformity.profile !0 {
+define void @branch_metadata(i1 %cond) !uniformity.profile !0 {
+; CHECK-LABEL: define void @branch_metadata(i1 %cond) !uniformity.profile !0 {
entry:
br i1 %cond, label %uniform, label %divergent, !block.uniformity.profile !0, !branch.uniformity.profile !0
; CHECK: br i1 %cond, label %uniform, label %divergent, !block.uniformity.profile !0, !branch.uniformity.profile !0
diff --git a/llvm/test/Verifier/block-uniformity-profile.ll b/llvm/test/Verifier/block-uniformity-profile.ll
index 7a8f59fc5dcf9..7084c2196d5d9 100644
--- a/llvm/test/Verifier/block-uniformity-profile.ll
+++ b/llvm/test/Verifier/block-uniformity-profile.ll
@@ -1,15 +1,10 @@
; RUN: not llvm-as -disable-output < %s 2>&1 | FileCheck %s
-; CHECK: block.uniformity.profile is only valid on function definitions and terminators
+; CHECK: block.uniformity.profile is only valid on terminators
declare !block.uniformity.profile !0 void @invalid_declaration()
-; CHECK: block.uniformity.profile must be an empty node
-define void @invalid_function_payload() !block.uniformity.profile !1 {
- ret void
-}
-
-; CHECK: block.uniformity.profile must be an empty node
-define void @invalid_duplicate_payload() !block.uniformity.profile !1 !block.uniformity.profile !0 {
+; CHECK: block.uniformity.profile is only valid on terminators
+define void @invalid_function() !block.uniformity.profile !0 {
ret void
}
@@ -18,13 +13,13 @@ define void @invalid_terminator_payload() {
ret void, !block.uniformity.profile !1
}
-; CHECK: block.uniformity.profile is only valid on function definitions and terminators
+; CHECK: block.uniformity.profile is only valid on terminators
define i32 @invalid_instruction(i32 %value) {
%sum = add i32 %value, 1, !block.uniformity.profile !0
ret i32 %sum
}
-; CHECK: block.uniformity.profile is only valid on function definitions and terminators
+; CHECK: block.uniformity.profile is only valid on terminators
; CHECK-NEXT: ptr @invalid_global
@invalid_global = global i32 0, !block.uniformity.profile !0
diff --git a/llvm/test/Verifier/uniformity-profile.ll b/llvm/test/Verifier/uniformity-profile.ll
new file mode 100644
index 0000000000000..e30e331c40ce6
--- /dev/null
+++ b/llvm/test/Verifier/uniformity-profile.ll
@@ -0,0 +1,27 @@
+; RUN: not llvm-as -disable-output < %s 2>&1 | FileCheck %s
+
+; CHECK: uniformity.profile is only valid on function definitions
+declare !uniformity.profile !0 void @invalid_declaration()
+
+; CHECK: uniformity.profile must be an empty node
+define void @invalid_function_payload() !uniformity.profile !1 {
+ ret void
+}
+
+; CHECK: uniformity.profile must be an empty node
+define void @invalid_duplicate_payload() !uniformity.profile !1 !uniformity.profile !0 {
+ ret void
+}
+
+; CHECK: uniformity.profile is only valid on function definitions
+define i32 @invalid_instruction(i32 %value) {
+ %sum = add i32 %value, 1, !uniformity.profile !0
+ ret i32 %sum
+}
+
+; CHECK: uniformity.profile is only valid on function definitions
+; CHECK-NEXT: ptr @invalid_global
+ at invalid_global = global i32 0, !uniformity.profile !0
+
+!0 = !{}
+!1 = !{i1 false}
>From 7b30e9690b5ff2524836150f227f3d0e2f8379d1 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Mon, 7 Sep 2026 00:32:58 -0400
Subject: [PATCH 2/2] [PGO] Load branch uniformity profiles and mark
availability
Uniformity profiles record block observations, but optimizations also
need information about branch decisions. A missing branch annotation
alone cannot tell them whether profile data is absent or no branches
are known to be uniform.
Mark functions when uniformity data is loaded and derive branch
annotations from blocks with a single conditional predecessor. Each
such block measures the lanes taking one outgoing edge. Mark a branch
as uniform only when all its instrumented outgoing edges are classified
as uniform, so one uniform edge cannot hide a divergent edge.
Replace existing branch annotations when loading the profile. This
clears an old uniform classification if the new data no longer supports
it. The function marker lets consumers use these results while keeping
their existing behavior when no uniformity profile is available.
---
.../Instrumentation/PGOInstrumentation.cpp | 21 ++++++++++++++++---
.../PGOInstrumentationTest.cpp | 18 +++++++++++++++-
2 files changed, 35 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 6d3126bcb8ac7..9a0203a03d5c7 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -1788,9 +1788,9 @@ void PGOUseFunc::setBlockUniformityAttribute() {
if (ProfileRecord.UniformityBits.empty())
return;
- // Annotate each uniform instrumented IR basic block so later codegen passes
- // (MachineFunction) can consume it without relying on fragile block numbering
- // heuristics.
+ // Mark the function as having uniformity profile, then annotate each uniform
+ // instrumented IR basic block so later codegen passes (MachineFunction) can
+ // consume it without relying on fragile block numbering heuristics.
// Metadata presence on a terminator means uniform; divergent blocks have no
// terminator metadata.
@@ -1799,14 +1799,29 @@ void PGOUseFunc::setBlockUniformityAttribute() {
LLVMContext &Ctx = F.getContext();
MDNode *UniformMD = MDNode::get(Ctx, {});
+ F.setMetadata(LLVMContext::MD_uniformity_profile, UniformMD);
+ DenseMap<CondBrInst *, bool> BranchUniformity;
for (size_t I = 0, E = InstrumentBBs.size(); I < E; ++I) {
BasicBlock *BB = InstrumentBBs[I];
if (!BB || !BB->getTerminator())
continue;
bool IsUniform = ProfileRecord.isBlockUniform(I);
+ // A counter placed in a block with a single conditional predecessor also
+ // measures the active lanes on that outgoing edge. Record the branch as
+ // uniform only when every instrumented outgoing edge is uniform.
+ if (BasicBlock *Pred = BB->getSinglePredecessor()) {
+ if (auto *Branch = dyn_cast<CondBrInst>(Pred->getTerminator())) {
+ auto It = BranchUniformity.try_emplace(Branch, true).first;
+ It->second &= IsUniform;
+ }
+ }
BB->getTerminator()->setMetadata(LLVMContext::MD_block_uniformity_profile,
IsUniform ? UniformMD : nullptr);
}
+ for (auto [Branch, IsUniform] : BranchUniformity) {
+ Branch->setMetadata(LLVMContext::MD_branch_uniformity_profile,
+ IsUniform ? UniformMD : nullptr);
+ }
LLVM_DEBUG({
dbgs() << "PGO: Set block uniformity profile for " << F.getName() << ": ";
diff --git a/llvm/unittests/Transforms/Instrumentation/PGOInstrumentationTest.cpp b/llvm/unittests/Transforms/Instrumentation/PGOInstrumentationTest.cpp
index 686410ac1468d..57ebb4b57dab2 100644
--- a/llvm/unittests/Transforms/Instrumentation/PGOInstrumentationTest.cpp
+++ b/llvm/unittests/Transforms/Instrumentation/PGOInstrumentationTest.cpp
@@ -197,7 +197,7 @@ struct PGOInstrumentationUseTest : Test, WithParamInterface<bool> {};
INSTANTIATE_TEST_SUITE_P(ExistingMetadata, PGOInstrumentationUseTest,
Values(false, true));
-TEST_P(PGOInstrumentationUseTest, BlockUniformityMetadataUsesPresence) {
+TEST_P(PGOInstrumentationUseTest, UniformityMetadataUsesPresence) {
static constexpr StringRef Code = R"(
define i32 @f(i1 %cond) {
entry:
@@ -278,15 +278,31 @@ TEST_P(PGOInstrumentationUseTest, BlockUniformityMetadataUsesPresence) {
ASSERT_THAT(UseFunction, NotNull());
if (GetParam()) {
MDNode *UniformMD = MDNode::get(Context, {});
+ UseFunction->setMetadata(LLVMContext::MD_uniformity_profile, UniformMD);
for (BasicBlock &BB : *UseFunction)
BB.getTerminator()->setMetadata(
LLVMContext::MD_block_uniformity_profile, UniformMD);
+ UseFunction->getEntryBlock().getTerminator()->setMetadata(
+ LLVMContext::MD_branch_uniformity_profile, UniformMD);
}
ModulePassManager UseMPM;
UseMPM.addPass(PGOInstrumentationUse("/profile.profdata", "", false, FS));
UseMPM.run(*UseModule, MAM);
EXPECT_FALSE(verifyModule(*UseModule, &errs()));
+ MDNode *FunctionMD =
+ UseFunction->getMetadata(LLVMContext::MD_uniformity_profile);
+ ASSERT_THAT(FunctionMD, NotNull());
+ EXPECT_EQ(FunctionMD->getNumOperands(), 0u);
+
+ auto *Branch =
+ cast<CondBrInst>(UseFunction->getEntryBlock().getTerminator());
+ MDNode *BranchMD =
+ Branch->getMetadata(LLVMContext::MD_branch_uniformity_profile);
+ EXPECT_EQ(BranchMD != nullptr, UniformityMask == 3);
+ if (BranchMD)
+ EXPECT_EQ(BranchMD->getNumOperands(), 0u);
+
for (unsigned I = 0; I < NumCounters; ++I) {
BasicBlock *BB = nullptr;
for (BasicBlock &Candidate : *UseFunction)
More information about the llvm-branch-commits
mailing list