[llvm-branch-commits] [llvm] [CHR] Use branch uniformity profiles to select scopes (PR #221517)
Yaxun Liu via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Sep 11 07:15:36 PDT 2026
https://github.com/yxsamliu updated https://github.com/llvm/llvm-project/pull/221517
>From 06dd54cdaa44fed2742a513aae5d5de5bfabf3d6 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Fri, 11 Sep 2026 09:34:15 -0400
Subject: [PATCH 1/3] [IR] Align uniformity metadata with RFC v2
---
llvm/docs/LangRef.md | 55 ++++++++++++-------
llvm/include/llvm/IR/FixedMetadataKinds.def | 1 +
llvm/lib/IR/Verifier.cpp | 19 ++++---
.../block-uniformity-profile-metadata.ll | 4 +-
.../test/Verifier/block-uniformity-profile.ll | 15 ++---
llvm/test/Verifier/uniformity-profile.ll | 27 +++++++++
6 files changed, 81 insertions(+), 40 deletions(-)
create mode 100644 llvm/test/Verifier/uniformity-profile.ll
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index ba562c88a7c22..b66146e9b3a8a 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -7897,21 +7897,38 @@ section is not marked as readable or writable and it uses the section flag
!0 = !{}
```
+(md_uniformity_profile)=
+
+#### '`uniformity.profile`' Metadata
+
+`uniformity.profile` metadata indicates that uniformity profile data was
+loaded for a function. It may be attached only to a function definition and
+must be an empty metadata node.
+
+This marker lets an optimization distinguish a function without uniformity
+profile data from one whose individual blocks or branches are not known to be
+uniform. Missing block or branch metadata may mean that the control flow was
+divergent, was not observed, or lacked enough profile information.
+
+```llvm
+define void @example() !uniformity.profile !0 {
+ ret void
+}
+
+!0 = !{}
+```
+
(md_block_uniformity_profile)=
#### '`block.uniformity.profile`' Metadata
`block.uniformity.profile` metadata records observed SIMT execution uniformity
-from an instrumentation profile. It may be attached to a function definition
-or a terminator instruction and must be an empty metadata node.
-
-On a function definition, this metadata indicates that a uniformity profile
-was loaded. On a terminator, it indicates that the profile classified the
-basic block as usually executing with all lanes active. This classification
-may tolerate some divergent executions in the profile. A function attachment
-without any annotated terminators can represent a profile in which no instrumented block was
-uniform. An unannotated block is not known to be uniform; it may be divergent
-or lack a uniformity observation.
+from an instrumentation profile. It may be attached only to a terminator
+instruction and must be an empty metadata node. It indicates that the profile
+classified the containing basic block as usually executing with all lanes
+active. This classification may tolerate some divergent executions in the
+profile. An unannotated block is not known to be uniform; it may be divergent,
+unobserved, or lack enough profile information.
Block uniformity does not describe the uniformity of a branch decision. For
example, all lanes may reach a block and then take different successors. See
@@ -7923,9 +7940,9 @@ guarantee uniform execution on other inputs and must not be used to justify
transformations that require uniformity for correctness.
```llvm
-define void @example(i1 %condition) !block.uniformity.profile !0 {
+define void @example(i1 %condition) !uniformity.profile !0 {
entry:
- br i1 %condition, label %then, label %else, !block.uniformity.profile !0
+ br i1 %condition, label %then, label %else, !block.uniformity.profile !0, !branch.uniformity.profile !0
then:
ret void
else:
@@ -7939,15 +7956,15 @@ else:
#### '`branch.uniformity.profile`' Metadata
-`branch.uniformity.profile` metadata records that a profile classified a
-conditional branch as usually uniform across lanes in a SIMT execution group.
-It may be attached only to a conditional branch instruction and must be an
-empty metadata node. A uniform decision means the lanes choose the same
-successor on that execution; the chosen successor can vary between executions.
-The profile classification may tolerate some divergent executions.
+`branch.uniformity.profile` metadata records a profile-derived hint that every
+instrumented successor used to classify a conditional branch usually executed
+with all lanes active. It may be attached only to a conditional branch
+instruction and must be an empty metadata node. The profiler does not directly
+observe every branch decision or whether only the currently active lanes chose
+the same successor.
An unannotated branch is not known to be uniform. Optimizations can use the
-function-level {ref}`block.uniformity.profile <md_block_uniformity_profile>`
+function-level {ref}`uniformity.profile <md_uniformity_profile>`
attachment to distinguish a function with uniformity profile information from
one without it.
diff --git a/llvm/include/llvm/IR/FixedMetadataKinds.def b/llvm/include/llvm/IR/FixedMetadataKinds.def
index 76e862eae3aee..54fbc1a3065be 100644
--- a/llvm/include/llvm/IR/FixedMetadataKinds.def
+++ b/llvm/include/llvm/IR/FixedMetadataKinds.def
@@ -69,3 +69,4 @@ LLVM_FIXED_MD_KIND(MD_callgraph, "callgraph", 54)
LLVM_FIXED_MD_KIND(MD_metadata_section_kind, "metadata_section_kind", 55)
LLVM_FIXED_MD_KIND(MD_branch_uniformity_profile, "branch.uniformity.profile",
56)
+LLVM_FIXED_MD_KIND(MD_uniformity_profile, "uniformity.profile", 57)
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 97560b4af30cb..751d2b1c941db 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -586,11 +586,11 @@ void Verifier::visitGlobalValue(const GlobalValue &GV) {
Check(!GO->hasMetadata(LLVMContext::MD_branch_uniformity_profile),
"branch.uniformity.profile is only valid on conditional branches",
GO);
- if (GO->hasMetadata(LLVMContext::MD_block_uniformity_profile))
+ Check(!GO->hasMetadata(LLVMContext::MD_block_uniformity_profile),
+ "block.uniformity.profile is only valid on terminators", GO);
+ if (GO->hasMetadata(LLVMContext::MD_uniformity_profile))
Check(isa<Function>(GO) && !GO->isDeclaration(),
- "block.uniformity.profile is only valid on function definitions "
- "and terminators",
- GO);
+ "uniformity.profile is only valid on function definitions", GO);
if (const MDNode *Associated =
GO->getMetadata(LLVMContext::MD_associated)) {
@@ -2769,9 +2769,9 @@ void Verifier::verifyUnknownProfileMetadata(MDNode *MD) {
void Verifier::verifyFunctionMetadata(
ArrayRef<std::pair<unsigned, MDNode *>> MDs) {
for (const auto &Pair : MDs) {
- if (Pair.first == LLVMContext::MD_block_uniformity_profile)
+ if (Pair.first == LLVMContext::MD_uniformity_profile)
Check(Pair.second->getNumOperands() == 0,
- "block.uniformity.profile must be an empty node", Pair.second);
+ "uniformity.profile must be an empty node", Pair.second);
if (Pair.first == LLVMContext::MD_prof) {
MDNode *MD = Pair.second;
Check(MD->getNumOperands() >= 2,
@@ -6061,13 +6061,14 @@ void Verifier::visitInstruction(Instruction &I) {
if (MDNode *MD = I.getMetadata(LLVMContext::MD_block_uniformity_profile)) {
Check(I.isTerminator(),
- "block.uniformity.profile is only valid on function definitions "
- "and terminators",
- &I);
+ "block.uniformity.profile is only valid on terminators", &I);
Check(MD->getNumOperands() == 0,
"block.uniformity.profile must be an empty node", &I, MD);
}
+ Check(!I.getMetadata(LLVMContext::MD_uniformity_profile),
+ "uniformity.profile is only valid on function definitions", &I);
+
if (MDNode *MD = I.getMetadata(LLVMContext::MD_branch_uniformity_profile)) {
Check(isa<CondBrInst>(I),
"branch.uniformity.profile is only valid on conditional branches",
diff --git a/llvm/test/Bitcode/block-uniformity-profile-metadata.ll b/llvm/test/Bitcode/block-uniformity-profile-metadata.ll
index a98f1cebda69d..bcdd48bf2b505 100644
--- a/llvm/test/Bitcode/block-uniformity-profile-metadata.ll
+++ b/llvm/test/Bitcode/block-uniformity-profile-metadata.ll
@@ -1,8 +1,8 @@
; RUN: llvm-as < %s | llvm-dis | FileCheck %s
; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
-define void @branch_metadata(i1 %cond) !block.uniformity.profile !0 {
-; CHECK-LABEL: define void @branch_metadata(i1 %cond) !block.uniformity.profile !0 {
+define void @branch_metadata(i1 %cond) !uniformity.profile !0 {
+; CHECK-LABEL: define void @branch_metadata(i1 %cond) !uniformity.profile !0 {
entry:
br i1 %cond, label %uniform, label %divergent, !block.uniformity.profile !0, !branch.uniformity.profile !0
; CHECK: br i1 %cond, label %uniform, label %divergent, !block.uniformity.profile !0, !branch.uniformity.profile !0
diff --git a/llvm/test/Verifier/block-uniformity-profile.ll b/llvm/test/Verifier/block-uniformity-profile.ll
index 7a8f59fc5dcf9..7084c2196d5d9 100644
--- a/llvm/test/Verifier/block-uniformity-profile.ll
+++ b/llvm/test/Verifier/block-uniformity-profile.ll
@@ -1,15 +1,10 @@
; RUN: not llvm-as -disable-output < %s 2>&1 | FileCheck %s
-; CHECK: block.uniformity.profile is only valid on function definitions and terminators
+; CHECK: block.uniformity.profile is only valid on terminators
declare !block.uniformity.profile !0 void @invalid_declaration()
-; CHECK: block.uniformity.profile must be an empty node
-define void @invalid_function_payload() !block.uniformity.profile !1 {
- ret void
-}
-
-; CHECK: block.uniformity.profile must be an empty node
-define void @invalid_duplicate_payload() !block.uniformity.profile !1 !block.uniformity.profile !0 {
+; CHECK: block.uniformity.profile is only valid on terminators
+define void @invalid_function() !block.uniformity.profile !0 {
ret void
}
@@ -18,13 +13,13 @@ define void @invalid_terminator_payload() {
ret void, !block.uniformity.profile !1
}
-; CHECK: block.uniformity.profile is only valid on function definitions and terminators
+; CHECK: block.uniformity.profile is only valid on terminators
define i32 @invalid_instruction(i32 %value) {
%sum = add i32 %value, 1, !block.uniformity.profile !0
ret i32 %sum
}
-; CHECK: block.uniformity.profile is only valid on function definitions and terminators
+; CHECK: block.uniformity.profile is only valid on terminators
; CHECK-NEXT: ptr @invalid_global
@invalid_global = global i32 0, !block.uniformity.profile !0
diff --git a/llvm/test/Verifier/uniformity-profile.ll b/llvm/test/Verifier/uniformity-profile.ll
new file mode 100644
index 0000000000000..e30e331c40ce6
--- /dev/null
+++ b/llvm/test/Verifier/uniformity-profile.ll
@@ -0,0 +1,27 @@
+; RUN: not llvm-as -disable-output < %s 2>&1 | FileCheck %s
+
+; CHECK: uniformity.profile is only valid on function definitions
+declare !uniformity.profile !0 void @invalid_declaration()
+
+; CHECK: uniformity.profile must be an empty node
+define void @invalid_function_payload() !uniformity.profile !1 {
+ ret void
+}
+
+; CHECK: uniformity.profile must be an empty node
+define void @invalid_duplicate_payload() !uniformity.profile !1 !uniformity.profile !0 {
+ ret void
+}
+
+; CHECK: uniformity.profile is only valid on function definitions
+define i32 @invalid_instruction(i32 %value) {
+ %sum = add i32 %value, 1, !uniformity.profile !0
+ ret i32 %sum
+}
+
+; CHECK: uniformity.profile is only valid on function definitions
+; CHECK-NEXT: ptr @invalid_global
+ at invalid_global = global i32 0, !uniformity.profile !0
+
+!0 = !{}
+!1 = !{i1 false}
>From 7b30e9690b5ff2524836150f227f3d0e2f8379d1 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Mon, 7 Sep 2026 00:32:58 -0400
Subject: [PATCH 2/3] [PGO] Load branch uniformity profiles and mark
availability
Uniformity profiles record block observations, but optimizations also
need information about branch decisions. A missing branch annotation
alone cannot tell them whether profile data is absent or no branches
are known to be uniform.
Mark functions when uniformity data is loaded and derive branch
annotations from blocks with a single conditional predecessor. Each
such block measures the lanes taking one outgoing edge. Mark a branch
as uniform only when all its instrumented outgoing edges are classified
as uniform, so one uniform edge cannot hide a divergent edge.
Replace existing branch annotations when loading the profile. This
clears an old uniform classification if the new data no longer supports
it. The function marker lets consumers use these results while keeping
their existing behavior when no uniformity profile is available.
---
.../Instrumentation/PGOInstrumentation.cpp | 21 ++++++++++++++++---
.../PGOInstrumentationTest.cpp | 18 +++++++++++++++-
2 files changed, 35 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 6d3126bcb8ac7..9a0203a03d5c7 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -1788,9 +1788,9 @@ void PGOUseFunc::setBlockUniformityAttribute() {
if (ProfileRecord.UniformityBits.empty())
return;
- // Annotate each uniform instrumented IR basic block so later codegen passes
- // (MachineFunction) can consume it without relying on fragile block numbering
- // heuristics.
+ // Mark the function as having uniformity profile, then annotate each uniform
+ // instrumented IR basic block so later codegen passes (MachineFunction) can
+ // consume it without relying on fragile block numbering heuristics.
// Metadata presence on a terminator means uniform; divergent blocks have no
// terminator metadata.
@@ -1799,14 +1799,29 @@ void PGOUseFunc::setBlockUniformityAttribute() {
LLVMContext &Ctx = F.getContext();
MDNode *UniformMD = MDNode::get(Ctx, {});
+ F.setMetadata(LLVMContext::MD_uniformity_profile, UniformMD);
+ DenseMap<CondBrInst *, bool> BranchUniformity;
for (size_t I = 0, E = InstrumentBBs.size(); I < E; ++I) {
BasicBlock *BB = InstrumentBBs[I];
if (!BB || !BB->getTerminator())
continue;
bool IsUniform = ProfileRecord.isBlockUniform(I);
+ // A counter placed in a block with a single conditional predecessor also
+ // measures the active lanes on that outgoing edge. Record the branch as
+ // uniform only when every instrumented outgoing edge is uniform.
+ if (BasicBlock *Pred = BB->getSinglePredecessor()) {
+ if (auto *Branch = dyn_cast<CondBrInst>(Pred->getTerminator())) {
+ auto It = BranchUniformity.try_emplace(Branch, true).first;
+ It->second &= IsUniform;
+ }
+ }
BB->getTerminator()->setMetadata(LLVMContext::MD_block_uniformity_profile,
IsUniform ? UniformMD : nullptr);
}
+ for (auto [Branch, IsUniform] : BranchUniformity) {
+ Branch->setMetadata(LLVMContext::MD_branch_uniformity_profile,
+ IsUniform ? UniformMD : nullptr);
+ }
LLVM_DEBUG({
dbgs() << "PGO: Set block uniformity profile for " << F.getName() << ": ";
diff --git a/llvm/unittests/Transforms/Instrumentation/PGOInstrumentationTest.cpp b/llvm/unittests/Transforms/Instrumentation/PGOInstrumentationTest.cpp
index 686410ac1468d..57ebb4b57dab2 100644
--- a/llvm/unittests/Transforms/Instrumentation/PGOInstrumentationTest.cpp
+++ b/llvm/unittests/Transforms/Instrumentation/PGOInstrumentationTest.cpp
@@ -197,7 +197,7 @@ struct PGOInstrumentationUseTest : Test, WithParamInterface<bool> {};
INSTANTIATE_TEST_SUITE_P(ExistingMetadata, PGOInstrumentationUseTest,
Values(false, true));
-TEST_P(PGOInstrumentationUseTest, BlockUniformityMetadataUsesPresence) {
+TEST_P(PGOInstrumentationUseTest, UniformityMetadataUsesPresence) {
static constexpr StringRef Code = R"(
define i32 @f(i1 %cond) {
entry:
@@ -278,15 +278,31 @@ TEST_P(PGOInstrumentationUseTest, BlockUniformityMetadataUsesPresence) {
ASSERT_THAT(UseFunction, NotNull());
if (GetParam()) {
MDNode *UniformMD = MDNode::get(Context, {});
+ UseFunction->setMetadata(LLVMContext::MD_uniformity_profile, UniformMD);
for (BasicBlock &BB : *UseFunction)
BB.getTerminator()->setMetadata(
LLVMContext::MD_block_uniformity_profile, UniformMD);
+ UseFunction->getEntryBlock().getTerminator()->setMetadata(
+ LLVMContext::MD_branch_uniformity_profile, UniformMD);
}
ModulePassManager UseMPM;
UseMPM.addPass(PGOInstrumentationUse("/profile.profdata", "", false, FS));
UseMPM.run(*UseModule, MAM);
EXPECT_FALSE(verifyModule(*UseModule, &errs()));
+ MDNode *FunctionMD =
+ UseFunction->getMetadata(LLVMContext::MD_uniformity_profile);
+ ASSERT_THAT(FunctionMD, NotNull());
+ EXPECT_EQ(FunctionMD->getNumOperands(), 0u);
+
+ auto *Branch =
+ cast<CondBrInst>(UseFunction->getEntryBlock().getTerminator());
+ MDNode *BranchMD =
+ Branch->getMetadata(LLVMContext::MD_branch_uniformity_profile);
+ EXPECT_EQ(BranchMD != nullptr, UniformityMask == 3);
+ if (BranchMD)
+ EXPECT_EQ(BranchMD->getNumOperands(), 0u);
+
for (unsigned I = 0; I < NumCounters; ++I) {
BasicBlock *BB = nullptr;
for (BasicBlock &Candidate : *UseFunction)
>From 35e7d0b96c6466fd0bc5a3e0d18fbb76377db58a Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Mon, 7 Sep 2026 00:32:58 -0400
Subject: [PATCH 3/3] [CHR] Use branch uniformity profiles to select scopes
Device PGO branch weights count lanes, so a divergent branch can look
strongly biased even when each wave executes both paths. CHR may then
duplicate control flow and increase register pressure without saving
branch work for the wave.
When a function has a uniformity profile, reject CHR scopes containing
branches that are not classified as uniform. This avoids transformations
whose expected benefit depends on lanes taking the same path together.
Keep the existing behavior for functions without uniformity data and
scopes containing only selects. Missing data is not evidence that a
branch is divergent, and selects do not split control flow.
---
.../ControlHeightReduction.cpp | 38 ++++
.../PGOProfile/chr-uniformity-profile.ll | 174 ++++++++++++++++++
2 files changed, 212 insertions(+)
create mode 100644 llvm/test/Transforms/PGOProfile/chr-uniformity-profile.ll
diff --git a/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp b/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp
index faf0e7debb3c1..e4859bd93203a 100644
--- a/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp
+++ b/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp
@@ -376,6 +376,11 @@ class CHR {
OptimizationRemarkEmitter &ORE;
CHRStats Stats;
+ // A function attachment indicates that uniformity profile data is available.
+ // This preserves CHR's existing behavior when no uniformity profile was
+ // collected.
+ bool HasUniformityProfile = F.getMetadata(LLVMContext::MD_uniformity_profile);
+
// All the true-biased regions in the function
DenseSet<Region *> TrueBiasedRegionsGlobal;
// All the false-biased regions in the function
@@ -1326,9 +1331,42 @@ static bool hasAtLeastTwoBiasedBranches(CHRScope *Scope) {
return NumBiased >= CHRMergeThreshold;
}
+static Instruction *
+findBranchWithoutUniformityProfile(const DenseSet<Region *> &Regions) {
+ for (Region *R : Regions) {
+ Instruction *Branch = R->getEntry()->getTerminator();
+ if (!Branch->getMetadata(LLVMContext::MD_branch_uniformity_profile))
+ return Branch;
+ }
+ return nullptr;
+}
+
void CHR::filterScopes(SmallVectorImpl<CHRScope *> &Input,
SmallVectorImpl<CHRScope *> &Output) {
for (CHRScope *Scope : Input) {
+ // Branch weights count individual lanes. A divergent branch can therefore
+ // look strongly biased even when every wave executes both paths. Avoid
+ // applying CHR to such a scope because duplicating its control flow may
+ // increase live ranges and register pressure without eliminating control
+ // flow for the wave. Select-only scopes are unaffected because branch
+ // uniformity profile describes only conditional branches.
+ if (HasUniformityProfile) {
+ Instruction *BranchWithoutUniformityProfile =
+ findBranchWithoutUniformityProfile(Scope->TrueBiasedRegions);
+ if (!BranchWithoutUniformityProfile)
+ BranchWithoutUniformityProfile =
+ findBranchWithoutUniformityProfile(Scope->FalseBiasedRegions);
+ if (BranchWithoutUniformityProfile) {
+ ORE.emit([&]() {
+ return OptimizationRemarkMissed(DEBUG_TYPE, "BranchNotKnownUniform",
+ BranchWithoutUniformityProfile)
+ << "drop scope containing a branch that is not known to be "
+ "uniform";
+ });
+ continue;
+ }
+ }
+
// Filter out the ones with only one region and no subs.
if (!hasAtLeastTwoBiasedBranches(Scope)) {
CHR_DEBUG(dbgs() << "Filtered out by biased branches truthy-regions "
diff --git a/llvm/test/Transforms/PGOProfile/chr-uniformity-profile.ll b/llvm/test/Transforms/PGOProfile/chr-uniformity-profile.ll
new file mode 100644
index 0000000000000..31ebc26aa3dbf
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/chr-uniformity-profile.ll
@@ -0,0 +1,174 @@
+; RUN: opt < %s -passes='require<profile-summary>,function(chr,instcombine,simplifycfg)' -S | FileCheck %s
+
+declare void @foo()
+
+; Preserve the existing behavior when a function has no uniformity profile.
+define void @no_profile(ptr %ptr) !prof !14 {
+; CHECK-LABEL: define void @no_profile(
+; CHECK: entry.split.nonchr:
+entry:
+ %value = load i32, ptr %ptr
+ %bit0 = and i32 %value, 1
+ %cond0 = icmp eq i32 %bit0, 0
+ br i1 %cond0, label %bb1, label %bb0, !prof !15
+
+bb0:
+ call void @foo()
+ br label %bb1
+
+bb1:
+ %bit1 = and i32 %value, 2
+ %cond1 = icmp eq i32 %bit1, 0
+ br i1 %cond1, label %exit, label %bb2, !prof !15
+
+bb2:
+ call void @foo()
+ br label %exit
+
+exit:
+ ret void
+}
+
+; Uniform branches remain eligible for CHR when uniformity profile is
+; present.
+define void @uniform(ptr %ptr) !prof !14 !uniformity.profile !16 {
+; CHECK-LABEL: define void @uniform(
+; CHECK: entry.split.nonchr:
+entry:
+ %value = load i32, ptr %ptr
+ %bit0 = and i32 %value, 1
+ %cond0 = icmp eq i32 %bit0, 0
+ br i1 %cond0, label %bb1, label %bb0, !prof !15,
+ !branch.uniformity.profile !16
+
+bb0:
+ call void @foo()
+ br label %bb1
+
+bb1:
+ %bit1 = and i32 %value, 2
+ %cond1 = icmp eq i32 %bit1, 0
+ br i1 %cond1, label %exit, label %bb2, !prof !15,
+ !branch.uniformity.profile !16
+
+bb2:
+ call void @foo()
+ br label %exit
+
+exit:
+ ret void
+}
+
+; With uniformity profile, a branch without branch-uniformity metadata
+; is not known to be uniform. Do not apply CHR to a scope containing one.
+define void @not_known_uniform(ptr %ptr) !prof !14 !uniformity.profile !16 {
+; CHECK-LABEL: define void @not_known_uniform(
+; CHECK-NOT: split
+; CHECK: ret void
+entry:
+ %value = load i32, ptr %ptr
+ %bit0 = and i32 %value, 1
+ %cond0 = icmp eq i32 %bit0, 0
+ br i1 %cond0, label %bb1, label %bb0, !prof !15,
+ !branch.uniformity.profile !16
+
+bb0:
+ call void @foo()
+ br label %bb1
+
+bb1:
+ %bit1 = and i32 %value, 2
+ %cond1 = icmp eq i32 %bit1, 0
+ br i1 %cond1, label %exit, label %bb2, !prof !15
+
+bb2:
+ call void @foo()
+ br label %exit
+
+exit:
+ ret void
+}
+
+; A scope with a branch that is not known uniform does not disable a separate
+; uniform scope in the same function.
+define void @per_scope(ptr %uniform_ptr, ptr %divergent_ptr) !prof !14 !uniformity.profile !16 {
+; CHECK-LABEL: define void @per_scope(
+; CHECK: entry.split.nonchr:
+; CHECK-NOT: after.uniform.split.nonchr:
+; CHECK: after.uniform:
+entry:
+ %uniform_value = load i32, ptr %uniform_ptr
+ %uniform_bit0 = and i32 %uniform_value, 1
+ %uniform_cond0 = icmp eq i32 %uniform_bit0, 0
+ br i1 %uniform_cond0, label %uniform.bb1, label %uniform.bb0, !prof !15,
+ !branch.uniformity.profile !16
+
+uniform.bb0:
+ call void @foo()
+ br label %uniform.bb1
+
+uniform.bb1:
+ %uniform_bit1 = and i32 %uniform_value, 2
+ %uniform_cond1 = icmp eq i32 %uniform_bit1, 0
+ br i1 %uniform_cond1, label %after.uniform, label %uniform.bb2, !prof !15,
+ !branch.uniformity.profile !16
+
+uniform.bb2:
+ call void @foo()
+ br label %after.uniform
+
+after.uniform:
+ %divergent_value = load i32, ptr %divergent_ptr
+ %divergent_bit0 = and i32 %divergent_value, 1
+ %divergent_cond0 = icmp eq i32 %divergent_bit0, 0
+ br i1 %divergent_cond0, label %divergent.bb1, label %divergent.bb0,
+ !prof !15
+
+divergent.bb0:
+ call void @foo()
+ br label %divergent.bb1
+
+divergent.bb1:
+ %divergent_bit1 = and i32 %divergent_value, 2
+ %divergent_cond1 = icmp eq i32 %divergent_bit1, 0
+ br i1 %divergent_cond1, label %exit, label %divergent.bb2, !prof !15
+
+divergent.bb2:
+ call void @foo()
+ br label %exit
+
+exit:
+ ret void
+}
+
+; Select-only scopes are not described by branch-uniformity profile and keep
+; their existing behavior.
+define i32 @select_only(i32 %value) !prof !14 !uniformity.profile !16 {
+; CHECK-LABEL: define i32 @select_only(
+; CHECK: entry.split.nonchr:
+entry:
+ %bit0 = and i32 %value, 1
+ %cond0 = icmp eq i32 %bit0, 0
+ %sum0 = select i1 %cond0, i32 %value, i32 42, !prof !15
+ %bit1 = and i32 %value, 2
+ %cond1 = icmp eq i32 %bit1, 0
+ %sum1 = select i1 %cond1, i32 %sum0, i32 43, !prof !15
+ ret i32 %sum1
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 1}
+!8 = !{!"NumFunctions", i64 1}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11}
+!11 = !{i32 999999, i64 1, i32 1}
+!14 = !{!"function_entry_count", i64 100}
+!15 = !{!"branch_weights", i32 0, i32 1}
+!16 = !{}
More information about the llvm-branch-commits
mailing list