[llvm-branch-commits] [llvm] [CHR] Use branch uniformity profiles to select scopes (PR #221517)
Yaxun Liu via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Oct 28 12:02:06 PDT 2026
https://github.com/yxsamliu updated https://github.com/llvm/llvm-project/pull/221517
>From b5fc30b4d6bf6fd2dcdef7bf2aeb872706544cb7 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Wed, 28 Oct 2026 16:47:54 +0000
Subject: [PATCH] [CHR] Use branch uniformity profiles to select scopes
Lane counts can make a divergent GPU branch look strongly biased. For
example, if 63 of 64 lanes take one path in every wave, that path has a
high count even though every wave also executes the other path. CHR
may duplicate control flow and increase register pressure without
avoiding the work on the other path.
When a function has uniformity.profile, require branch.uniformity.profile
on each biased branch used to form a candidate CHR scope. Skip the
scope if any such branch lacks the hint. Missing hints can also mean
missing observations, so this is a conservative profitability decision.
Functions without uniformity data and scopes containing only selects
retain their existing behavior.
---
.../ControlHeightReduction.cpp | 37 ++++
.../chr-uniformity-profile-amdgpu.ll | 5 +
.../PGOProfile/chr-uniformity-profile.ll | 174 ++++++++++++++++++
3 files changed, 216 insertions(+)
create mode 100644 llvm/test/Transforms/PGOProfile/chr-uniformity-profile-amdgpu.ll
create mode 100644 llvm/test/Transforms/PGOProfile/chr-uniformity-profile.ll
diff --git a/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp b/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp
index 8984e6cbdc6f62..67f4ffa543b9c4 100644
--- a/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp
+++ b/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp
@@ -356,6 +356,11 @@ class CHR {
OptimizationRemarkEmitter &ORE;
CHRStats Stats;
+ // A function attachment indicates that uniformity profile data is available.
+ // This preserves CHR's existing behavior when no uniformity profile was
+ // collected.
+ bool HasUniformityProfile = F.getMetadata(LLVMContext::MD_uniformity_profile);
+
// All the true-biased regions in the function
DenseSet<Region *> TrueBiasedRegionsGlobal;
// All the false-biased regions in the function
@@ -1307,9 +1312,41 @@ static bool hasAtLeastTwoBiasedBranches(const InstrumentationOptions &Opts,
return NumBiased >= Opts.chr_merge_threshold;
}
+static Instruction *
+findBranchWithoutUniformityProfile(const DenseSet<Region *> &Regions) {
+ for (Region *R : Regions) {
+ Instruction *Branch = R->getEntry()->getTerminator();
+ if (!Branch->getMetadata(LLVMContext::MD_branch_uniformity_profile))
+ return Branch;
+ }
+ return nullptr;
+}
+
void CHR::filterScopes(SmallVectorImpl<CHRScope *> &Input,
SmallVectorImpl<CHRScope *> &Output) {
for (CHRScope *Scope : Input) {
+ // Branch weights count individual lanes. A divergent branch can therefore
+ // look strongly biased even when every wave executes both paths. Avoid
+ // applying CHR to such a scope because duplicating its control flow may
+ // increase live ranges and register pressure without eliminating control
+ // flow for the wave.
+ if (HasUniformityProfile) {
+ Instruction *BranchWithoutUniformityProfile =
+ findBranchWithoutUniformityProfile(Scope->TrueBiasedRegions);
+ if (!BranchWithoutUniformityProfile)
+ BranchWithoutUniformityProfile =
+ findBranchWithoutUniformityProfile(Scope->FalseBiasedRegions);
+ if (BranchWithoutUniformityProfile) {
+ ORE.emit([&]() {
+ return OptimizationRemarkMissed(DEBUG_TYPE, "BranchNotKnownUniform",
+ BranchWithoutUniformityProfile)
+ << "drop scope containing a branch that is not known to be "
+ "uniform";
+ });
+ continue;
+ }
+ }
+
// Filter out the ones with only one region and no subs.
if (!hasAtLeastTwoBiasedBranches(Opts, Scope)) {
CHR_DEBUG(dbgs() << "Filtered out by biased branches truthy-regions "
diff --git a/llvm/test/Transforms/PGOProfile/chr-uniformity-profile-amdgpu.ll b/llvm/test/Transforms/PGOProfile/chr-uniformity-profile-amdgpu.ll
new file mode 100644
index 00000000000000..f8552e2ce64196
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/chr-uniformity-profile-amdgpu.ll
@@ -0,0 +1,5 @@
+; REQUIRES: amdgpu-registered-target
+; RUN: opt < %S/chr-uniformity-profile.ll -mtriple=amdgpu9.50-amd-amdhsa -passes='default<O3>' -S | FileCheck %S/chr-uniformity-profile.ll
+
+; Run the shared scope checks with the AMDGPU O3 pipeline. Keep this separately
+; gated so the target-independent checks also run in builds without AMDGPU.
diff --git a/llvm/test/Transforms/PGOProfile/chr-uniformity-profile.ll b/llvm/test/Transforms/PGOProfile/chr-uniformity-profile.ll
new file mode 100644
index 00000000000000..056637ffca88b0
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/chr-uniformity-profile.ll
@@ -0,0 +1,174 @@
+; RUN: opt < %s -passes='require<profile-summary>,function(chr,instcombine,simplifycfg)' -S | FileCheck %s
+; RUN: opt < %s -passes='default<O3>' -S | FileCheck %s
+
+declare void @foo()
+
+; Preserve the existing behavior when a function has no uniformity profile.
+define void @no_profile(ptr %ptr) !prof !14 {
+; CHECK-LABEL: define void @no_profile(
+; CHECK: entry.split.nonchr:
+entry:
+ %value = load i32, ptr %ptr
+ %bit0 = and i32 %value, 1
+ %cond0 = icmp eq i32 %bit0, 0
+ br i1 %cond0, label %bb1, label %bb0, !prof !15
+
+bb0:
+ call void @foo()
+ br label %bb1
+
+bb1:
+ %bit1 = and i32 %value, 2
+ %cond1 = icmp eq i32 %bit1, 0
+ br i1 %cond1, label %exit, label %bb2, !prof !15
+
+bb2:
+ call void @foo()
+ br label %exit
+
+exit:
+ ret void
+}
+
+; Uniform branches remain eligible for CHR when uniformity profile is
+; present.
+define void @uniform(ptr %ptr) !prof !14 !uniformity.profile !16 {
+; CHECK-LABEL: define void @uniform(
+; CHECK: entry.split.nonchr:
+entry:
+ %value = load i32, ptr %ptr
+ %bit0 = and i32 %value, 1
+ %cond0 = icmp eq i32 %bit0, 0
+ br i1 %cond0, label %bb1, label %bb0, !prof !15,
+ !branch.uniformity.profile !16
+
+bb0:
+ call void @foo()
+ br label %bb1
+
+bb1:
+ %bit1 = and i32 %value, 2
+ %cond1 = icmp eq i32 %bit1, 0
+ br i1 %cond1, label %exit, label %bb2, !prof !15,
+ !branch.uniformity.profile !16
+
+bb2:
+ call void @foo()
+ br label %exit
+
+exit:
+ ret void
+}
+
+; With uniformity profile, a branch without branch-uniformity metadata
+; is not known to be uniform. Do not apply CHR to a scope containing one.
+define void @not_known_uniform(ptr %ptr) !prof !14 !uniformity.profile !16 {
+; CHECK-LABEL: define void @not_known_uniform(
+; CHECK-NOT: split
+; CHECK: ret void
+entry:
+ %value = load i32, ptr %ptr
+ %bit0 = and i32 %value, 1
+ %cond0 = icmp eq i32 %bit0, 0
+ br i1 %cond0, label %bb1, label %bb0, !prof !15,
+ !branch.uniformity.profile !16
+
+bb0:
+ call void @foo()
+ br label %bb1
+
+bb1:
+ %bit1 = and i32 %value, 2
+ %cond1 = icmp eq i32 %bit1, 0
+ br i1 %cond1, label %exit, label %bb2, !prof !15
+
+bb2:
+ call void @foo()
+ br label %exit
+
+exit:
+ ret void
+}
+
+; A scope with a branch that is not known uniform does not disable a separate
+; uniform scope in the same function.
+define void @per_scope(ptr %uniform_ptr, ptr %divergent_ptr) !prof !14 !uniformity.profile !16 {
+; CHECK-LABEL: define void @per_scope(
+; CHECK: entry.split.nonchr:
+; CHECK-NOT: after.uniform.split.nonchr:
+; CHECK: after.uniform:
+entry:
+ %uniform_value = load i32, ptr %uniform_ptr
+ %uniform_bit0 = and i32 %uniform_value, 1
+ %uniform_cond0 = icmp eq i32 %uniform_bit0, 0
+ br i1 %uniform_cond0, label %uniform.bb1, label %uniform.bb0, !prof !15,
+ !branch.uniformity.profile !16
+
+uniform.bb0:
+ call void @foo()
+ br label %uniform.bb1
+
+uniform.bb1:
+ %uniform_bit1 = and i32 %uniform_value, 2
+ %uniform_cond1 = icmp eq i32 %uniform_bit1, 0
+ br i1 %uniform_cond1, label %after.uniform, label %uniform.bb2, !prof !15,
+ !branch.uniformity.profile !16
+
+uniform.bb2:
+ call void @foo()
+ br label %after.uniform
+
+after.uniform:
+ %divergent_value = load i32, ptr %divergent_ptr
+ %divergent_bit0 = and i32 %divergent_value, 1
+ %divergent_cond0 = icmp eq i32 %divergent_bit0, 0
+ br i1 %divergent_cond0, label %divergent.bb1, label %divergent.bb0,
+ !prof !15
+
+divergent.bb0:
+ call void @foo()
+ br label %divergent.bb1
+
+divergent.bb1:
+ %divergent_bit1 = and i32 %divergent_value, 2
+ %divergent_cond1 = icmp eq i32 %divergent_bit1, 0
+ br i1 %divergent_cond1, label %exit, label %divergent.bb2, !prof !15
+
+divergent.bb2:
+ call void @foo()
+ br label %exit
+
+exit:
+ ret void
+}
+
+; Scopes containing only non-Boolean selects keep their existing behavior.
+define i32 @select_only(i32 %value) !prof !14 !uniformity.profile !16 {
+; CHECK-LABEL: define i32 @select_only(
+; CHECK: entry.split.nonchr:
+entry:
+ %bit0 = and i32 %value, 1
+ %cond0 = icmp eq i32 %bit0, 0
+ %sum0 = select i1 %cond0, i32 %value, i32 42, !prof !15
+ %bit1 = and i32 %value, 2
+ %cond1 = icmp eq i32 %bit1, 0
+ %sum1 = select i1 %cond1, i32 %sum0, i32 43, !prof !15
+ ret i32 %sum1
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 1}
+!8 = !{!"NumFunctions", i64 1}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11}
+!11 = !{i32 999999, i64 1, i32 1}
+!14 = !{!"function_entry_count", i64 100}
+!15 = !{!"branch_weights", i32 0, i32 1}
+!16 = !{}
More information about the llvm-branch-commits
mailing list