[llvm-branch-commits] [llvm] [Transforms] Maintain uniformity profile hints across CFG changes (PR #225284)
Yaxun Liu via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sat Nov 14 05:00:47 PST 2026
https://github.com/yxsamliu updated https://github.com/llvm/llvm-project/pull/225284
>From aea4c719a215a3e3999a076967d2c31bf44b753e Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Wed, 28 Oct 2026 16:47:54 +0000
Subject: [PATCH] [Transforms] Maintain uniformity profile hints across CFG
changes
Uniformity hints describe a block's execution population or evidence
for a branch decision. Preserve them when those properties survive a
CFG rewrite, and clear them when the rewrite changes their meaning.
For example, folding a conditional branch to an unconditional branch
can preserve the block hint, but must drop the branch hint. Combining
two predicates or splitting aggregate executions across predecessor
paths cannot reuse the old branch classification.
---
llvm/lib/Transforms/Scalar/JumpThreading.cpp | 15 ++-
llvm/lib/Transforms/Scalar/StructurizeCFG.cpp | 20 ++-
llvm/lib/Transforms/Utils/BasicBlockUtils.cpp | 4 +
llvm/lib/Transforms/Utils/Local.cpp | 10 +-
llvm/lib/Transforms/Utils/SimplifyCFG.cpp | 12 ++
.../InstCombine/cfg-uniformity-profile.ll | 18 +++
.../JumpThreading/cfg-uniformity-profile.ll | 74 +++++++++++
.../SimplifyCFG/cfg-uniformity-bypass.ll | 59 +++++++++
.../SimplifyCFG/cfg-uniformity-profile.ll | 120 ++++++++++++++++++
.../StructurizeCFG/uniformity-profile.ll | 117 +++++++++++++++++
10 files changed, 444 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/Transforms/InstCombine/cfg-uniformity-profile.ll
create mode 100644 llvm/test/Transforms/JumpThreading/cfg-uniformity-profile.ll
create mode 100644 llvm/test/Transforms/SimplifyCFG/cfg-uniformity-bypass.ll
create mode 100644 llvm/test/Transforms/SimplifyCFG/cfg-uniformity-profile.ll
create mode 100644 llvm/test/Transforms/StructurizeCFG/uniformity-profile.ll
diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index 7f49cb33901d27..0d78ea1c91e598 100644
--- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
@@ -2709,6 +2709,14 @@ bool JumpThreadingPass::duplicateCondBranchOnPHIIntoPred(
identifyNoAliasScopesToClone(BI, BB->end(), NoAliasScopes);
cloneNoAliasScopes(NoAliasScopes, ClonedScopes, "thread", Context);
+ // Both the duplicate and the remaining original execute on subsets of the
+ // old predecessor paths. Neither can inherit their aggregate uniformity
+ // classification. Clear it before cloning the terminator.
+ BB->getTerminator()->setMetadata(LLVMContext::MD_block_uniformity_profile,
+ nullptr);
+ BB->getTerminator()->setMetadata(LLVMContext::MD_branch_uniformity_profile,
+ nullptr);
+
// Clone the non-phi instructions of BB into PredBB, keeping track of the
// mapping and using it to remap operands in the cloned instructions.
for (; BI != BB->end(); ++BI) {
@@ -2776,8 +2784,13 @@ bool JumpThreadingPass::duplicateCondBranchOnPHIIntoPred(
// that we nuked.
BB->removePredecessor(PredBB, true);
- // Remove the unconditional branch at the end of the PredBB block.
+ // The new terminator still describes PredBB's executions, so preserve its
+ // block hint rather than the classification of the duplicated block.
+ MDNode *PredUniformity =
+ OldPredBranch->getMetadata(LLVMContext::MD_block_uniformity_profile);
OldPredBranch->eraseFromParent();
+ PredBB->getTerminator()->setMetadata(LLVMContext::MD_block_uniformity_profile,
+ PredUniformity);
DTU->applyUpdatesPermissive(Updates);
BasicBlock *ThreadBB = PredBB;
diff --git a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
index d94052d2837fb4..ef136ffb835abc 100644
--- a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
+++ b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
@@ -277,6 +277,11 @@ class StructurizeCFG {
BBSet Visited;
BBSet FlowSet;
+ // The terminator carries a profile of the block's execution, independently
+ // of its branch condition. Keep it when replacing the terminator of the
+ // same block, but do not propagate it to newly created flow blocks.
+ DenseMap<BasicBlock *, MDNode *> BlockUniformityProfiles;
+
SmallVector<WeakVH, 8> AffectedPhis;
BBPhiMap DeletedPhis;
BB2BBVecMap AddedPhis;
@@ -1001,6 +1006,9 @@ DebugLoc StructurizeCFG::killTerminator(BasicBlock *BB) {
for (BasicBlock *Succ : successors(BB))
delPhiValues(BB, Succ);
+ if (MDNode *MD = Term->getMetadata(LLVMContext::MD_block_uniformity_profile))
+ BlockUniformityProfiles[BB] = MD;
+
DebugLoc DL = Term->getDebugLoc();
Term->eraseFromParent();
return DL;
@@ -1071,8 +1079,13 @@ std::pair<BasicBlock *, DebugLoc> StructurizeCFG::needPrefix(bool NeedEmpty) {
if (!PrevNode->isSubRegion()) {
DebugLoc DL = killTerminator(Entry);
- if (!NeedEmpty || Entry->getFirstInsertionPt() == Entry->end())
+ if (!NeedEmpty || Entry->getFirstInsertionPt() == Entry->end()) {
+ // An empty prefix reused as a loop header also executes on backedges.
+ // Its old execution profile does not describe those additional visits.
+ if (NeedEmpty)
+ BlockUniformityProfiles.erase(Entry);
return {Entry, DL};
+ }
}
// create a new flow node
@@ -1355,7 +1368,12 @@ bool StructurizeCFG::run(Region *R, DominatorTree *DT,
simplifyAffectedPhis();
rebuildSSA();
+ for (auto [BB, MD] : BlockUniformityProfiles)
+ BB->getTerminator()->setMetadata(LLVMContext::MD_block_uniformity_profile,
+ MD);
+
// Cleanup
+ BlockUniformityProfiles.clear();
Order.clear();
Visited.clear();
DeletedPhis.clear();
diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
index c731e92a360d25..f275675da9d3bc 100644
--- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -362,6 +362,10 @@ bool llvm::MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU,
// Update branch in the predecessor.
PredBB_BI->setSuccessor(FallThruPath, NewSucc);
} else {
+ // The merged block retains the predecessor's executions. Its terminator
+ // changes, but its block-uniformity classification still applies.
+ STI->copyMetadata(*PTI, {LLVMContext::MD_block_uniformity_profile});
+
// Delete the unconditional branch from the predecessor.
PredBB->back().eraseFromParent();
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index 8b6d53483dca04..1e6e0529d878e4 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -155,7 +155,8 @@ bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions,
// Transfer the metadata to the new branch instruction.
NewBI->copyMetadata(*BI, {LLVMContext::MD_loop, LLVMContext::MD_dbg,
- LLVMContext::MD_annotation});
+ LLVMContext::MD_annotation,
+ LLVMContext::MD_block_uniformity_profile});
Value *Cond = BI->getCondition();
BI->eraseFromParent();
@@ -180,7 +181,8 @@ bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions,
// Transfer the metadata to the new branch instruction.
NewBI->copyMetadata(*BI, {LLVMContext::MD_loop, LLVMContext::MD_dbg,
- LLVMContext::MD_annotation});
+ LLVMContext::MD_annotation,
+ LLVMContext::MD_block_uniformity_profile});
BI->eraseFromParent();
if (DTU)
@@ -275,7 +277,8 @@ bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions,
if (TheOnlyDest) {
// Insert the new branch.
IRBuilder<> Builder(SI);
- Builder.CreateBr(TheOnlyDest);
+ Builder.CreateBr(TheOnlyDest)
+ ->copyMetadata(*SI, {LLVMContext::MD_block_uniformity_profile});
BasicBlock *BB = SI->getParent();
SmallPtrSet<BasicBlock *, 8> RemovedSuccessors;
@@ -319,6 +322,7 @@ bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions,
// Insert the new branch.
CondBrInst *NewBr = Builder.CreateCondBr(
Cond, FirstCase.getCaseSuccessor(), SI->getDefaultDest());
+ NewBr->copyMetadata(*SI, {LLVMContext::MD_block_uniformity_profile});
SmallVector<uint32_t> Weights;
if (extractBranchWeights(*SI, Weights) && Weights.size() == 2) {
uint32_t DefWeight = Weights[0];
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index dabc1c709c02ba..ab10bb3e5ddb2e 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -4275,6 +4275,12 @@ static bool performBranchToCommonDestFolding(CondBrInst *BI, CondBrInst *PBI,
setExplicitlyUnknownBranchWeightsIfProfiled(*PBI, DEBUG_TYPE);
}
+ // The new predicate has different successor evidence. BB, if it survives,
+ // also executes only on the remaining predecessor paths.
+ PBI->setMetadata(LLVMContext::MD_branch_uniformity_profile, nullptr);
+ BI->setMetadata(LLVMContext::MD_block_uniformity_profile, nullptr);
+ BI->setMetadata(LLVMContext::MD_branch_uniformity_profile, nullptr);
+
// Now, update the CFG.
PBI->setSuccessor(PBI->getSuccessor(0) != BB, UniqueSucc);
@@ -5008,6 +5014,12 @@ static bool SimplifyCondBranchToCondBranch(CondBrInst *PBI, CondBrInst *BI,
Value *Cond =
createLogicalOp(Builder, Instruction::Or, PBICond, BICond, "brmerge");
+ // The new predicate has different successor evidence. BB, if it survives,
+ // also executes only on the remaining predecessor paths.
+ PBI->setMetadata(LLVMContext::MD_branch_uniformity_profile, nullptr);
+ BI->setMetadata(LLVMContext::MD_block_uniformity_profile, nullptr);
+ BI->setMetadata(LLVMContext::MD_branch_uniformity_profile, nullptr);
+
// Modify PBI to branch on the new condition to the new dests.
PBI->setCondition(Cond);
PBI->setSuccessor(0, CommonDest);
diff --git a/llvm/test/Transforms/InstCombine/cfg-uniformity-profile.ll b/llvm/test/Transforms/InstCombine/cfg-uniformity-profile.ll
new file mode 100644
index 00000000000000..5f3228b55bd3c8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/cfg-uniformity-profile.ll
@@ -0,0 +1,18 @@
+; RUN: opt -S -passes=instcombine -verify-each < %s | FileCheck %s
+
+; Inversion preserves the observed block/successor events and swaps weights.
+; CHECK-LABEL: define i32 @invert(
+; CHECK: br i1 %cond, label %no, label %yes, !prof ![[W:[0-9]+]], !block.uniformity.profile ![[U:[0-9]+]], !branch.uniformity.profile ![[U]]{{$}}
+; CHECK: ![[W]] = !{!"branch_weights", i32 10, i32 90}
+
+define i32 @invert(i1 %cond) !uniformity.profile !0 {
+entry:
+ %neg = xor i1 %cond, true
+ br i1 %neg, label %yes, label %no, !prof !1, !block.uniformity.profile !0, !branch.uniformity.profile !0
+yes:
+ ret i32 1
+no:
+ ret i32 0
+}
+!0 = !{}
+!1 = !{!"branch_weights", i32 90, i32 10}
diff --git a/llvm/test/Transforms/JumpThreading/cfg-uniformity-profile.ll b/llvm/test/Transforms/JumpThreading/cfg-uniformity-profile.ll
new file mode 100644
index 00000000000000..16d1e391c0cf4e
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/cfg-uniformity-profile.ll
@@ -0,0 +1,74 @@
+; RUN: opt -S -passes=jump-threading -verify-each < %s | FileCheck %s
+
+; A reconverged block may have a full-wave profile even when either predecessor
+; executes with a partial mask. Drop both hints on the duplicate and the original.
+; CHECK-LABEL: define void @clone_phi(
+; CHECK: br i1 %x, label %yes, label %no, !prof ![[W:[0-9]+]]{{$}}
+; CHECK: br i1 %y, label %yes, label %no, !prof ![[W]]{{$}}
+define void @clone_phi(i1 %dispatch, i1 %x, i1 %y) !uniformity.profile !0 {
+entry:
+ br i1 %dispatch, label %left, label %right
+left:
+ call void @left_call()
+ br label %join
+right:
+ call void @right_call()
+ br label %join
+join:
+ %test = phi i1 [ %x, %left ], [ %y, %right ]
+ br i1 %test, label %yes, label %no, !prof !1, !block.uniformity.profile !0, !branch.uniformity.profile !0
+yes:
+ call void @a()
+ ret void
+no:
+ call void @b()
+ ret void
+}
+
+; A positive predecessor hint still describes the duplicate's block after an
+; unconditional split and merge. It does not classify its new branch decision.
+; CHECK-LABEL: define void @annotated_predecessor(
+; CHECK: br i1 %x, label %yes, label %no, !prof ![[W]], !block.uniformity.profile ![[U:[0-9]+]]{{$}}
+; CHECK: br i1 %y, label %yes, label %no, !prof ![[W]]{{$}}
+define void @annotated_predecessor(i1 %dispatch, i1 %x, i1 %y) !uniformity.profile !0 {
+entry:
+ br i1 %dispatch, label %left, label %right
+left:
+ call void @left_call()
+ br label %join, !block.uniformity.profile !0
+right:
+ call void @right_call()
+ br label %join
+join:
+ %test = phi i1 [ %x, %left ], [ %y, %right ]
+ br i1 %test, label %yes, label %no, !prof !1, !block.uniformity.profile !0, !branch.uniformity.profile !0
+yes:
+ call void @a()
+ ret void
+no:
+ call void @b()
+ ret void
+}
+
+; An unchanged branch retains both classifications.
+; CHECK-LABEL: define void @unchanged(
+; CHECK: br i1 %cond, label %yes, label %no, !prof ![[W]], !block.uniformity.profile ![[U]], !branch.uniformity.profile ![[U]]{{$}}
+define void @unchanged(i1 %cond) !uniformity.profile !0 {
+entry:
+ br i1 %cond, label %yes, label %no, !prof !1, !block.uniformity.profile !0, !branch.uniformity.profile !0
+yes:
+ call void @a()
+ ret void
+no:
+ call void @b()
+ ret void
+}
+
+declare void @left_call()
+declare void @right_call()
+declare void @a()
+declare void @b()
+
+; CHECK: ![[W]] = !{!"branch_weights", i32 90, i32 10}
+!0 = !{}
+!1 = !{!"branch_weights", i32 90, i32 10}
diff --git a/llvm/test/Transforms/SimplifyCFG/cfg-uniformity-bypass.ll b/llvm/test/Transforms/SimplifyCFG/cfg-uniformity-bypass.ll
new file mode 100644
index 00000000000000..4a68bece9b3bc6
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/cfg-uniformity-bypass.ll
@@ -0,0 +1,59 @@
+; RUN: opt -S -passes=simplifycfg -verify-each < %s | FileCheck %s
+
+; Cover both condition folding implementations. The bypassed block survives on
+; the right path, where it must lose its old block and branch classifications.
+
+declare void @a()
+declare void @b()
+declare void @left_call()
+declare void @right_call()
+
+; CHECK-LABEL: define void @computed(
+; CHECK: br i1 %or.cond, label %yes, label %no, !prof ![[COMPUTED:[0-9]+]], !block.uniformity.profile ![[U:[0-9]+]]{{$}}
+; CHECK: br i1 %y.old, label %yes, label %no, !prof ![[ORIGINAL:[0-9]+]]{{$}}
+define void @computed(i1 %dispatch, i1 %x, i32 %value) !uniformity.profile !0 {
+entry:
+ br i1 %dispatch, label %left, label %right
+left:
+ call void @left_call()
+ br i1 %x, label %inner, label %no, !prof !1, !block.uniformity.profile !0, !branch.uniformity.profile !0
+right:
+ call void @right_call()
+ br label %inner
+inner:
+ %y = icmp eq i32 %value, 0
+ br i1 %y, label %yes, label %no, !prof !1, !block.uniformity.profile !0, !branch.uniformity.profile !0
+yes:
+ call void @a()
+ ret void
+no:
+ call void @b()
+ ret void
+}
+
+; CHECK-LABEL: define void @empty(
+; CHECK: br i1 %brmerge, label %no, label %yes, !prof ![[EMPTY:[0-9]+]], !block.uniformity.profile ![[U]]{{$}}
+; CHECK: br i1 %y, label %yes, label %no, !prof ![[ORIGINAL]]{{$}}
+define void @empty(i1 %dispatch, i1 %x, i1 %y) !uniformity.profile !0 {
+entry:
+ br i1 %dispatch, label %left, label %right
+left:
+ call void @left_call()
+ br i1 %x, label %inner, label %no, !prof !1, !block.uniformity.profile !0, !branch.uniformity.profile !0
+right:
+ call void @right_call()
+ br label %inner
+inner:
+ br i1 %y, label %yes, label %no, !prof !1, !block.uniformity.profile !0, !branch.uniformity.profile !0
+yes:
+ call void @a()
+ ret void
+no:
+ call void @b()
+ ret void
+}
+; CHECK-DAG: ![[COMPUTED]] = !{!"branch_weights", i32 8100, i32 1900}
+; CHECK-DAG: ![[ORIGINAL]] = !{!"branch_weights", i32 90, i32 10}
+; CHECK-DAG: ![[EMPTY]] = !{!"branch_weights", i32 1900, i32 8100}
+!0 = !{}
+!1 = !{!"branch_weights", i32 90, i32 10}
diff --git a/llvm/test/Transforms/SimplifyCFG/cfg-uniformity-profile.ll b/llvm/test/Transforms/SimplifyCFG/cfg-uniformity-profile.ll
new file mode 100644
index 00000000000000..592d0003910cc7
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/cfg-uniformity-profile.ll
@@ -0,0 +1,120 @@
+; RUN: opt -S -passes=simplifycfg -verify-each < %s | FileCheck %s
+
+; Folding two predicates must drop the old branch hint, but retain the
+; unchanged entry block hint and recompute branch weights.
+; CHECK-LABEL: define void @fold(
+; CHECK: br i1 %brmerge, label %no, label %yes, !prof ![[FOLD:[0-9]+]], !block.uniformity.profile ![[U:[0-9]+]]{{$}}
+define void @fold(i1 %x, i1 %y) !uniformity.profile !0 {
+entry:
+ br i1 %x, label %inner, label %no, !prof !1, !block.uniformity.profile !0, !branch.uniformity.profile !0
+inner:
+ br i1 %y, label %yes, label %no, !prof !1
+yes:
+ call void @a()
+ ret void
+no:
+ call void @b()
+ ret void
+}
+
+; Missing outer weights do not justify preserving branch uniformity.
+; CHECK-LABEL: define void @fold_proxy(
+; CHECK: br i1 %brmerge, label %no, label %yes, !prof !{{[0-9]+}}, !block.uniformity.profile ![[U]]{{$}}
+define void @fold_proxy(i1 %x, i1 %y) !uniformity.profile !0 {
+entry:
+ br i1 %x, label %inner, label %no, !block.uniformity.profile !0, !branch.uniformity.profile !0
+inner:
+ br i1 %y, label %yes, label %no, !prof !2, !block.uniformity.profile !0
+yes:
+ call void @a()
+ ret void
+no:
+ call void @b()
+ ret void
+}
+
+; Preserve block execution information through branch replacement and merge.
+; CHECK-LABEL: define void @constant(
+; CHECK: call void @a()
+; CHECK-NEXT: ret void, !block.uniformity.profile ![[U]]{{$}}
+define void @constant(i1 %dispatch) !uniformity.profile !0 {
+entry:
+ call void @a()
+ br i1 true, label %yes, label %no, !prof !1, !block.uniformity.profile !0, !branch.uniformity.profile !0
+yes:
+ ret void
+no:
+ call void @a()
+ ret void
+}
+
+; Switch weights are default/case; branch weights are true/false.
+; A new conditional branch must not acquire a branch-uniformity hint.
+; CHECK-LABEL: define void @switch_one(
+; CHECK: br i1 %cond, label %yes, label %no, !prof ![[SWITCH:[0-9]+]], !block.uniformity.profile ![[U]]{{$}}
+define void @switch_one(i32 %x) !uniformity.profile !0 {
+entry:
+ switch i32 %x, label %no [
+ i32 0, label %yes
+ ], !prof !3, !block.uniformity.profile !0
+yes:
+ call void @a()
+ ret void
+no:
+ call void @b()
+ ret void
+}
+
+; Folding a constant switch retains block metadata through the return merge.
+; CHECK-LABEL: define void @constant_switch(
+; CHECK: call void @a()
+; CHECK-NEXT: ret void, !block.uniformity.profile ![[U]]{{$}}
+define void @constant_switch() !uniformity.profile !0 {
+entry:
+ call void @a()
+ switch i32 0, label %no [i32 0, label %yes], !prof !3, !block.uniformity.profile !0
+yes:
+ ret void
+no:
+ call void @b()
+ ret void
+}
+
+; An unknown block must not become annotated merely because profile data exists.
+; CHECK-LABEL: define void @unknown(
+; CHECK: call void @a()
+; CHECK-NEXT: ret void{{$}}
+define void @unknown() !uniformity.profile !0 {
+entry:
+ call void @a()
+ br i1 true, label %yes, label %no
+yes:
+ ret void
+no:
+ call void @b()
+ ret void
+}
+
+; Both arms lead to the same block: the branch hint becomes obsolete, while
+; the predecessor block's classification survives replacement and merging.
+; CHECK-LABEL: define void @same_destination(
+; CHECK: call void @a()
+; CHECK-NEXT: ret void, !block.uniformity.profile ![[U]]{{$}}
+define void @same_destination(i1 %cond) !uniformity.profile !0 {
+entry:
+ call void @a()
+ br i1 %cond, label %exit, label %exit, !prof !1, !block.uniformity.profile !0, !branch.uniformity.profile !0
+exit:
+ ret void
+}
+
+declare void @a()
+declare void @b()
+
+; CHECK-DAG: ![[U]] = !{}
+; CHECK-DAG: ![[FOLD]] = !{!"branch_weights", i32 1900, i32 8100}
+; CHECK-DAG: ![[SWITCH]] = !{!"branch_weights", i32 90, i32 10}
+!0 = !{}
+!1 = !{!"branch_weights", i32 90, i32 10}
+!2 = !{!"branch_weights", i32 50, i32 50}
+!3 = !{!"branch_weights", i32 10, i32 90}
diff --git a/llvm/test/Transforms/StructurizeCFG/uniformity-profile.ll b/llvm/test/Transforms/StructurizeCFG/uniformity-profile.ll
new file mode 100644
index 00000000000000..df78540888fdff
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/uniformity-profile.ll
@@ -0,0 +1,117 @@
+; RUN: opt -S -passes=structurizecfg -structurizecfg-skip-uniform-regions=false -verify-each %s | FileCheck %s --implicit-check-not=block.uniformity.profile --implicit-check-not=branch.uniformity.profile
+;
+; Block profiles describe executions of the original block, not the branch
+; condition. Replacing a terminator must keep the block hint, without copying
+; the old branch hint onto a synthesized condition or annotating a new Flow.
+
+; CHECK-LABEL: define void @diamond(
+; CHECK: entry:
+; CHECK: br i1 {{.*}}, label {{.*}}, label {{.*}}, !block.uniformity.profile ![[MD:[0-9]+]]{{$}}
+; CHECK: then:
+; CHECK-NEXT: store i32 1, ptr %out
+; CHECK-NEXT: br label {{.*}}, !block.uniformity.profile ![[MD]]{{$}}
+; CHECK: exit:
+; CHECK-NEXT: ret void, !block.uniformity.profile ![[MD]]{{$}}
+define void @diamond(i1 %cond, ptr %out) !uniformity.profile !0 {
+entry:
+ br i1 %cond, label %then, label %else, !block.uniformity.profile !0, !branch.uniformity.profile !0
+then:
+ store i32 1, ptr %out
+ br label %exit, !block.uniformity.profile !0
+else:
+ store i32 2, ptr %out
+ br label %exit
+exit:
+ ret void, !block.uniformity.profile !0
+}
+
+; A reused empty prefix gains loop-backedge executions and must lose its hint.
+; CHECK-LABEL: define void @loop_prefix(
+; CHECK: prefix:{{.*}}preds = %{{.*}}, %entry
+; CHECK: a:
+; CHECK-NEXT: store i32 3, ptr %out
+; CHECK-NEXT: br i1 {{.*}}, label {{.*}}, label {{.*}}, !block.uniformity.profile ![[MD]]{{$}}
+define void @loop_prefix(i1 %enter, i1 %stop, i1 %again, ptr %out) !uniformity.profile !0 {
+entry:
+ br label %prefix
+prefix:
+ br i1 %enter, label %a, label %exit.a, !block.uniformity.profile !0, !branch.uniformity.profile !0
+a:
+ store i32 3, ptr %out
+ br i1 %stop, label %exit.a, label %b, !block.uniformity.profile !0
+b:
+ store i32 4, ptr %out
+ br i1 %again, label %a, label %exit.b
+exit.a:
+ store i32 5, ptr %out
+ br label %exit
+exit.b:
+ store i32 6, ptr %out
+ br label %exit
+exit:
+ ret void
+}
+
+; A nonempty prefix keeps its original executions. needPrefix removes its
+; terminator before changeExit removes it again; preserve the saved hint.
+; CHECK-LABEL: define void @nonempty_loop_prefix(
+; CHECK: prefix:{{.*}}preds = %entry
+; CHECK-NEXT: store i32 2, ptr %out
+; CHECK-NEXT: br label {{.*}}, !block.uniformity.profile ![[MD]]{{$}}
+; CHECK: a:
+; CHECK-NEXT: store i32 3, ptr %out
+; CHECK-NEXT: br i1 {{.*}}, label {{.*}}, label {{.*}}, !block.uniformity.profile ![[MD]]{{$}}
+define void @nonempty_loop_prefix(i1 %enter, i1 %stop, i1 %again, ptr %out) !uniformity.profile !0 {
+entry:
+ br label %prefix
+prefix:
+ store i32 2, ptr %out
+ br i1 %enter, label %a, label %exit.a, !block.uniformity.profile !0, !branch.uniformity.profile !0
+a:
+ store i32 3, ptr %out
+ br i1 %stop, label %exit.a, label %b, !block.uniformity.profile !0
+b:
+ store i32 4, ptr %out
+ br i1 %again, label %a, label %exit.b
+exit.a:
+ store i32 5, ptr %out
+ br label %exit
+exit.b:
+ store i32 6, ptr %out
+ br label %exit
+exit:
+ ret void
+}
+
+; Preserve the inner block profiles when a containing region is rewritten.
+; CHECK-LABEL: define void @nested(
+; CHECK: outer.then:
+; CHECK-NEXT: br i1 {{.*}}, label {{.*}}, label {{.*}}, !block.uniformity.profile ![[MD]]{{$}}
+; CHECK: inner.then:
+; CHECK-NEXT: store i32 5, ptr %out
+; CHECK-NEXT: br label {{.*}}, !block.uniformity.profile ![[MD]]{{$}}
+; CHECK: inner.exit:
+; CHECK-NEXT: store i32 7, ptr %out
+; CHECK-NEXT: br label {{.*}}, !block.uniformity.profile ![[MD]]{{$}}
+define void @nested(i1 %outer, i1 %inner, ptr %out) !uniformity.profile !0 {
+entry:
+ br i1 %outer, label %outer.then, label %outer.else
+outer.then:
+ br i1 %inner, label %inner.then, label %inner.else, !block.uniformity.profile !0, !branch.uniformity.profile !0
+inner.then:
+ store i32 5, ptr %out
+ br label %inner.exit, !block.uniformity.profile !0
+inner.else:
+ store i32 6, ptr %out
+ br label %inner.exit
+inner.exit:
+ store i32 7, ptr %out
+ br label %exit, !block.uniformity.profile !0
+outer.else:
+ store i32 8, ptr %out
+ br label %exit
+exit:
+ ret void
+}
+
+!0 = !{}
More information about the llvm-branch-commits
mailing list