[llvm] [IR] Define uniformity profile metadata (PR #221628)
Yaxun Liu via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 11 06:34:53 PDT 2026
https://github.com/yxsamliu updated https://github.com/llvm/llvm-project/pull/221628
>From 7a45eb1ec1f9a9d7a9347cdf7f9847c6fd9bada7 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Mon, 7 Sep 2026 00:32:57 -0400
Subject: [PATCH 1/2] [IR] Define uniformity profile metadata
Branch weights show how often each path is taken, but not whether lanes
take the same path together. GPU optimizations need this distinction to
judge whether a change is likely to help. They also need to distinguish
missing uniformity data from a profile with no known uniform behavior.
Define and document branch.uniformity.profile for branch decisions and
block.uniformity.profile for block execution. Use the block metadata on
functions to mark available profile data, and verify where these
annotations are allowed. These are hints for optimization decisions;
they do not prove that execution is always uniform.
Represent uniform classifications with empty nodes and omit the other
annotations. Update the existing block producer and upgrade old boolean
metadata when reading bitcode, so newly generated IR and older bitcode
both follow the same rules.
---
llvm/docs/LangRef.md | 64 ++++++++++
llvm/include/llvm/IR/FixedMetadataKinds.def | 2 +
llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 12 ++
llvm/lib/IR/Verifier.cpp | 29 +++++
.../Instrumentation/PGOInstrumentation.cpp | 18 ++-
.../block-uniformity-profile-metadata.ll | 18 ++-
.../block-uniformity-profile-upgrade.ll | 23 ++++
.../block-uniformity-profile-upgrade.ll.bc | Bin 0 -> 2164 bytes
.../branch-uniformity-profile-metadata.ll | 15 +++
.../test/Verifier/block-uniformity-profile.ll | 32 +++++
.../Verifier/branch-uniformity-profile.ll | 52 ++++++++
.../PGOInstrumentationTest.cpp | 117 ++++++++++++++++++
12 files changed, 361 insertions(+), 21 deletions(-)
create mode 100644 llvm/test/Bitcode/block-uniformity-profile-upgrade.ll
create mode 100644 llvm/test/Bitcode/block-uniformity-profile-upgrade.ll.bc
create mode 100644 llvm/test/Bitcode/branch-uniformity-profile-metadata.ll
create mode 100644 llvm/test/Verifier/block-uniformity-profile.ll
create mode 100644 llvm/test/Verifier/branch-uniformity-profile.ll
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index 14caff88243c1..ba562c88a7c22 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -7897,6 +7897,70 @@ section is not marked as readable or writable and it uses the section flag
!0 = !{}
```
+(md_block_uniformity_profile)=
+
+#### '`block.uniformity.profile`' Metadata
+
+`block.uniformity.profile` metadata records observed SIMT execution uniformity
+from an instrumentation profile. It may be attached to a function definition
+or a terminator instruction and must be an empty metadata node.
+
+On a function definition, this metadata indicates that a uniformity profile
+was loaded. On a terminator, it indicates that the profile classified the
+basic block as usually executing with all lanes active. This classification
+may tolerate some divergent executions in the profile. A function attachment
+without any annotated terminators can represent a profile in which no instrumented block was
+uniform. An unannotated block is not known to be uniform; it may be divergent
+or lack a uniformity observation.
+
+Block uniformity does not describe the uniformity of a branch decision. For
+example, all lanes may reach a block and then take different successors. See
+{ref}`branch.uniformity.profile <md_branch_uniformity_profile>` for branch
+decisions.
+
+This metadata is a profile hint for optimization profitability. It does not
+guarantee uniform execution on other inputs and must not be used to justify
+transformations that require uniformity for correctness.
+
+```llvm
+define void @example(i1 %condition) !block.uniformity.profile !0 {
+entry:
+ br i1 %condition, label %then, label %else, !block.uniformity.profile !0
+then:
+ ret void
+else:
+ ret void
+}
+
+!0 = !{}
+```
+
+(md_branch_uniformity_profile)=
+
+#### '`branch.uniformity.profile`' Metadata
+
+`branch.uniformity.profile` metadata records that a profile classified a
+conditional branch as usually uniform across lanes in a SIMT execution group.
+It may be attached only to a conditional branch instruction and must be an
+empty metadata node. A uniform decision means the lanes choose the same
+successor on that execution; the chosen successor can vary between executions.
+The profile classification may tolerate some divergent executions.
+
+An unannotated branch is not known to be uniform. Optimizations can use the
+function-level {ref}`block.uniformity.profile <md_block_uniformity_profile>`
+attachment to distinguish a function with uniformity profile information from
+one without it.
+
+This metadata is a profile hint for optimization profitability. It does not
+guarantee that the branch is uniform on other inputs and must not be used to
+justify transformations that require uniformity for correctness.
+
+```llvm
+br i1 %condition, label %then, label %else, !branch.uniformity.profile !0
+
+!0 = !{}
+```
+
#### '`unpredictable`' Metadata
`unpredictable` metadata may be attached to any branch, select, or switch
diff --git a/llvm/include/llvm/IR/FixedMetadataKinds.def b/llvm/include/llvm/IR/FixedMetadataKinds.def
index fee5e22cd61f4..76e862eae3aee 100644
--- a/llvm/include/llvm/IR/FixedMetadataKinds.def
+++ b/llvm/include/llvm/IR/FixedMetadataKinds.def
@@ -67,3 +67,5 @@ LLVM_FIXED_MD_KIND(MD_mem_cache_hint, "mem.cache_hint", 52)
LLVM_FIXED_MD_KIND(MD_block_uniformity_profile, "block.uniformity.profile", 53)
LLVM_FIXED_MD_KIND(MD_callgraph, "callgraph", 54)
LLVM_FIXED_MD_KIND(MD_metadata_section_kind, "metadata_section_kind", 55)
+LLVM_FIXED_MD_KIND(MD_branch_uniformity_profile, "branch.uniformity.profile",
+ 56)
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 35e71c93c0109..b297e8a505425 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -7213,6 +7213,18 @@ Error BitcodeReader::materialize(GlobalValue *GV) {
}
for (auto &I : make_early_inc_range(instructions(F))) {
+ // Older block uniformity profiles used an i1 payload instead of presence.
+ if (MDNode *MD = I.getMetadata(LLVMContext::MD_block_uniformity_profile)) {
+ if (MD->getNumOperands() == 1) {
+ auto *Uniform =
+ mdconst::dyn_extract_or_null<ConstantInt>(MD->getOperand(0));
+ if (Uniform && Uniform->getType()->isIntegerTy(1))
+ I.setMetadata(LLVMContext::MD_block_uniformity_profile,
+ Uniform->isOne() ? MDNode::get(I.getContext(), {})
+ : nullptr);
+ }
+ }
+
// "Upgrade" older incorrect branch weights by dropping them.
if (auto *MD = I.getMetadata(LLVMContext::MD_prof)) {
if (MD->getOperand(0) != nullptr && isa<MDString>(MD->getOperand(0))) {
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 20df60ed61da3..97560b4af30cb 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -583,6 +583,15 @@ void Verifier::visitGlobalValue(const GlobalValue &GV) {
"Global is external, but doesn't have external or weak linkage!", &GV);
if (const auto *GO = dyn_cast<GlobalObject>(&GV)) {
+ Check(!GO->hasMetadata(LLVMContext::MD_branch_uniformity_profile),
+ "branch.uniformity.profile is only valid on conditional branches",
+ GO);
+ if (GO->hasMetadata(LLVMContext::MD_block_uniformity_profile))
+ Check(isa<Function>(GO) && !GO->isDeclaration(),
+ "block.uniformity.profile is only valid on function definitions "
+ "and terminators",
+ GO);
+
if (const MDNode *Associated =
GO->getMetadata(LLVMContext::MD_associated)) {
Check(Associated->getNumOperands() == 1,
@@ -2760,6 +2769,9 @@ void Verifier::verifyUnknownProfileMetadata(MDNode *MD) {
void Verifier::verifyFunctionMetadata(
ArrayRef<std::pair<unsigned, MDNode *>> MDs) {
for (const auto &Pair : MDs) {
+ if (Pair.first == LLVMContext::MD_block_uniformity_profile)
+ Check(Pair.second->getNumOperands() == 0,
+ "block.uniformity.profile must be an empty node", Pair.second);
if (Pair.first == LLVMContext::MD_prof) {
MDNode *MD = Pair.second;
Check(MD->getNumOperands() >= 2,
@@ -6047,6 +6059,23 @@ void Verifier::visitInstruction(Instruction &I) {
if (MDNode *MD = I.getMetadata(LLVMContext::MD_prof))
visitProfMetadata(I, MD);
+ if (MDNode *MD = I.getMetadata(LLVMContext::MD_block_uniformity_profile)) {
+ Check(I.isTerminator(),
+ "block.uniformity.profile is only valid on function definitions "
+ "and terminators",
+ &I);
+ Check(MD->getNumOperands() == 0,
+ "block.uniformity.profile must be an empty node", &I, MD);
+ }
+
+ if (MDNode *MD = I.getMetadata(LLVMContext::MD_branch_uniformity_profile)) {
+ Check(isa<CondBrInst>(I),
+ "branch.uniformity.profile is only valid on conditional branches",
+ &I);
+ Check(MD->getNumOperands() == 0,
+ "branch.uniformity.profile must be an empty node", &I, MD);
+ }
+
if (MDNode *MD = I.getMetadata(LLVMContext::MD_memprof))
visitMemProfMetadata(I, MD);
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 06ce7525a0fd8..6d3126bcb8ac7 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -1788,28 +1788,24 @@ void PGOUseFunc::setBlockUniformityAttribute() {
if (ProfileRecord.UniformityBits.empty())
return;
- // Annotate uniformity on each instrumented IR basic block so later codegen
- // passes (MachineFunction) can consume it without relying on fragile block
- // numbering heuristics.
- //
- // Metadata kind: LLVMContext::MD_block_uniformity_profile
- // Payload: i1 (true = uniform, false = divergent)
+ // Annotate each uniform instrumented IR basic block so later codegen passes
+ // (MachineFunction) can consume it without relying on fragile block numbering
+ // heuristics.
+ // Metadata presence on a terminator means uniform; divergent blocks have no
+ // terminator metadata.
std::vector<BasicBlock *> InstrumentBBs;
FuncInfo.getInstrumentBBs(InstrumentBBs);
LLVMContext &Ctx = F.getContext();
- Type *Int1Ty = Type::getInt1Ty(Ctx);
-
+ MDNode *UniformMD = MDNode::get(Ctx, {});
for (size_t I = 0, E = InstrumentBBs.size(); I < E; ++I) {
BasicBlock *BB = InstrumentBBs[I];
if (!BB || !BB->getTerminator())
continue;
bool IsUniform = ProfileRecord.isBlockUniform(I);
- auto *MD = MDNode::get(
- Ctx, ConstantAsMetadata::get(ConstantInt::get(Int1Ty, IsUniform)));
BB->getTerminator()->setMetadata(LLVMContext::MD_block_uniformity_profile,
- MD);
+ IsUniform ? UniformMD : nullptr);
}
LLVM_DEBUG({
diff --git a/llvm/test/Bitcode/block-uniformity-profile-metadata.ll b/llvm/test/Bitcode/block-uniformity-profile-metadata.ll
index b69956fa26622..a98f1cebda69d 100644
--- a/llvm/test/Bitcode/block-uniformity-profile-metadata.ll
+++ b/llvm/test/Bitcode/block-uniformity-profile-metadata.ll
@@ -1,21 +1,19 @@
; RUN: llvm-as < %s | llvm-dis | FileCheck %s
; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
-define void @branch_metadata(i1 %cond) {
-; CHECK-LABEL: define void @branch_metadata(
+define void @branch_metadata(i1 %cond) !block.uniformity.profile !0 {
+; CHECK-LABEL: define void @branch_metadata(i1 %cond) !block.uniformity.profile !0 {
entry:
- br i1 %cond, label %uniform, label %divergent, !block.uniformity.profile !0
-; CHECK: br i1 %cond, label %uniform, label %divergent, !block.uniformity.profile !0
+ br i1 %cond, label %uniform, label %divergent, !block.uniformity.profile !0, !branch.uniformity.profile !0
+; CHECK: br i1 %cond, label %uniform, label %divergent, !block.uniformity.profile !0, !branch.uniformity.profile !0
uniform:
ret void
divergent:
- br label %uniform, !block.uniformity.profile !1
-; CHECK: br label %uniform, !block.uniformity.profile !1
+ br label %uniform, !block.uniformity.profile !0
+; CHECK: br label %uniform, !block.uniformity.profile !0
}
-; CHECK: !0 = !{i1 true}
-; CHECK: !1 = !{i1 false}
-!0 = !{i1 true}
-!1 = !{i1 false}
+; CHECK: !0 = !{}
+!0 = !{}
diff --git a/llvm/test/Bitcode/block-uniformity-profile-upgrade.ll b/llvm/test/Bitcode/block-uniformity-profile-upgrade.ll
new file mode 100644
index 0000000000000..b1104198a3f49
--- /dev/null
+++ b/llvm/test/Bitcode/block-uniformity-profile-upgrade.ll
@@ -0,0 +1,23 @@
+; RUN: llvm-dis < %s.bc | FileCheck %s
+; RUN: opt -passes=verify -disable-output < %s.bc
+
+; The bitcode was generated before block.uniformity.profile used presence-only
+; metadata. Preserve true hints as empty nodes and remove false hints.
+
+define void @branch_metadata(i1 %cond) {
+; CHECK-LABEL: define void @branch_metadata(
+entry:
+ br i1 %cond, label %uniform, label %divergent, !block.uniformity.profile !0
+; CHECK: br i1 %cond, label %uniform, label %divergent, !block.uniformity.profile [[UNIFORM:![0-9]+]]
+
+uniform:
+ ret void
+
+divergent:
+ br label %uniform, !block.uniformity.profile !1
+; CHECK: br label %uniform{{$}}
+}
+
+; CHECK: [[UNIFORM]] = !{}
+!0 = !{i1 true}
+!1 = !{i1 false}
diff --git a/llvm/test/Bitcode/block-uniformity-profile-upgrade.ll.bc b/llvm/test/Bitcode/block-uniformity-profile-upgrade.ll.bc
new file mode 100644
index 0000000000000000000000000000000000000000..3a71ec69197b67a07b80db8e2301f73e2072aff0
GIT binary patch
literal 2164
zcmZ`*e at q+K9e?I9=M?kq=-7h8-PIRpgjAB;Ot#tBu52!0RAQ>jGL31Q;E3bEnjgk-
zz>q1ja~_vuUgu0!=08#uRy1wWCb<=VcvIA1ju3~qh at xy`EkbrNG^t3i2$>?S()K;*
zADgsUy7&Cf?|tw4{rvczTg9oBZ`2^9M+o&Ag@*6^;Dg`(;-8bR*JT at y>MVN18xi`K
z4x!329+knKdbr3){dIS?{Iv?&V*8aYYJ6F5FmdHS?+qJ|m9<{0k2dU$X3gG4KP}v-
z at p|d&J@;Bijn#YJvU;QKpT?{Aw|9n(CjLg5H*2VFu({es>H9j1CE9K|?m2FLH|=T*
zM2sfOD0Sl%i?`PF8n$?6x&m>&zAgST&9(%ySKf;AM at M7jmLmvVHA8NY(|%t${eJ7^
zdvqrZ at quYi`wM#xh_m`u%K?LNpDtN%xJFl%cR(cw?c+P}U03eXFaP$(Kikh-y8lW4
z$Ormg-~O<F at K4`U-dU3Zf4}2<b3%yxrb65OKR#1*W#~VD3zMkfn}!;WMkrgR+riSp
zB at L$Pj(BX(fp5>2i=9<5gx;$~=oQGI*$`%C3XHEt=e#0XqiU=*Ob&<0^T`3 at I-(xa
zcsq=njjh~thMga0e1d*wn_QP8oLSKsBb>QVEa)dM8pz>x#qTFa{KQ~L at tYM>t{QK|
zaY3NY3e;)|mpr(co?%ybW{GE~CC(>YwcnEM3!-h3aNLp|QNo at p)eQEK7yHO>^(g at u
z!-_wy_<P8~Mso1E9>NZVm2QJFP^$!z%5XB$^rVEJ<ngm9yde<1*>R>g&g6E+Wr>;N
znVASXCsq74F50r9Ex9wMRck`DULouW^)Nj2i_7*0ggvDi at AfPHE+ycXgMDPEM+w#{
z!7;@&|6S?{qP9G^7{{wFs$j+&P=&dXUCwaRZf=ohmLuG9o+%dC&vl%y_6=)FY`H1h
zbDoAT3+YjYLS!iUM#pxX(h%T&!p%v{L!Mp9u$Oe4mN3tjq>AG|ldU<`o)Vn^$QRR&
zqG-PYOEq at q1*lXRf_x0*NFy0YMt-zP<Bci2=D|;e`plx6D;D-H71$L>pxtCo$<9f+
zE~;8%qAjO7(ivM;g&G~X_Q?SQ30VyQfL)5|lUcmcLp{z{l+4tcK&_dne|Yd&8m|f%
zmQV8)kMg+a!Ow;IPd6lHI>Ig$_HOX(I?vAVEHL<xf5~Tl+kQ)QrU_d at w&l8R0sBzw
z2;|wW4EYt){m<~GKo#=%k&AjX22W9S#QEQmP)m0bq at p1lTmUK~QMWCx0(4GbD(9LR
zW9P=%yqjCeFusNp0xoF$QV?WK5PA7REqT6O=^j&l8iup)C3KFM>PV>81+fL>lmo<!
z?fh3;B at R-Z&#;}58ehjL0dJbA&3whChuYRc;kA6lx|u30K(X(-<iS32*r1p?_oY=k
z=x{=`<^V&&7MC44!U13=RAXD!EJzm=HO^d$a9cXyarFHPP$0wsE;h~hbD>_60p!q_
zG7{G$YU+R>K%zaS9$rxGe-LeHH8xwd2-QW{MTuDi at wm7-D6{cg9&h&Gl7Kg8pp|N_
zwbt$^|G6IU9JVR}P~+>&gbWI4NrOVlS|(Hn#LSAX=F-J6W+}s5lKxv-1vU>}h;WPJ
z%(~R}b at DThR(LblvTH7gUrPoXm2Q^;{vAnP|4YQWlWI%a?M#a;NiDd2f^Zh)x+xhP
zYManpW4wcRahP{=(>iWJ8ofY1_TX(A7ok_s_%>)sz}s=Wna8(NSW}UpW(atYAM6ZX
z4k-gkjru_Hqcyj6lCa$*91rBW1o7&WXoG%|AZ%&36~0W!&V*c-60J~gj?g;a at 1bXX
zALhzyez2&O9O`n#z$ApNDfEBLpE-s>*=s#JV;--1 at UDk%m+)4p@`;B6BUQ$<P$6=-
zQPC?}OG=B*6k%_&E`zHxEJV&px+ByY0{>G5P+_U^sfz+V75sQDHD*=<;P5dxYkMHu
zV}KFiZ0ehrn2*C;5gLTV0nLjAc3E;a<8?oNM&m7)hSU}ucTttKA<&BEOwHtez{RZS
z at 7iJ_RNk^A+L}_c5msxynJVZwvoJ=uhdeW at W9K5wE)!p4N-k=P#?M^T!&JCS84#3U
zR|$|wYX~}SYO(_bTNh+UO1*ql>x{FI611 at 0#Kx=E`IkO+Gb<4mp^xCbp}kJXoCDN$
zbElj!*#S*6u4*qVsSzQd0o5BuO9IU0aDAnmPnr$Wqvj>r084b3E?d8xcOh-D*YBz7
zv{s)lyB9uw)JuQpC8GHws;=f}^D!+9()A*wk^94nF&sT8d&nK85gIV>-nU?Se-u6X
z at -9L@hW`LB!u}VD#6_4;Z$LVGqObeR$)Mzwf|BLPA>oj4sz2C!`rL^(4_-Li?>iSb
e(;pl@=ntIp^`Aa@@PhwTK=PhEbo#V*9Q_;mHXY^w
literal 0
HcmV?d00001
diff --git a/llvm/test/Bitcode/branch-uniformity-profile-metadata.ll b/llvm/test/Bitcode/branch-uniformity-profile-metadata.ll
new file mode 100644
index 0000000000000..e84b1b02eb804
--- /dev/null
+++ b/llvm/test/Bitcode/branch-uniformity-profile-metadata.ll
@@ -0,0 +1,15 @@
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s
+; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+
+define void @uniform_branch(i1 %condition) {
+; CHECK-LABEL: define void @uniform_branch(
+; CHECK: br i1 %condition, label %then, label %else, !branch.uniformity.profile !0
+ br i1 %condition, label %then, label %else, !branch.uniformity.profile !0
+then:
+ ret void
+else:
+ ret void
+}
+
+; CHECK: !0 = !{}
+!0 = !{}
diff --git a/llvm/test/Verifier/block-uniformity-profile.ll b/llvm/test/Verifier/block-uniformity-profile.ll
new file mode 100644
index 0000000000000..7a8f59fc5dcf9
--- /dev/null
+++ b/llvm/test/Verifier/block-uniformity-profile.ll
@@ -0,0 +1,32 @@
+; RUN: not llvm-as -disable-output < %s 2>&1 | FileCheck %s
+
+; CHECK: block.uniformity.profile is only valid on function definitions and terminators
+declare !block.uniformity.profile !0 void @invalid_declaration()
+
+; CHECK: block.uniformity.profile must be an empty node
+define void @invalid_function_payload() !block.uniformity.profile !1 {
+ ret void
+}
+
+; CHECK: block.uniformity.profile must be an empty node
+define void @invalid_duplicate_payload() !block.uniformity.profile !1 !block.uniformity.profile !0 {
+ ret void
+}
+
+; CHECK: block.uniformity.profile must be an empty node
+define void @invalid_terminator_payload() {
+ ret void, !block.uniformity.profile !1
+}
+
+; CHECK: block.uniformity.profile is only valid on function definitions and terminators
+define i32 @invalid_instruction(i32 %value) {
+ %sum = add i32 %value, 1, !block.uniformity.profile !0
+ ret i32 %sum
+}
+
+; CHECK: block.uniformity.profile is only valid on function definitions and terminators
+; CHECK-NEXT: ptr @invalid_global
+ at invalid_global = global i32 0, !block.uniformity.profile !0
+
+!0 = !{}
+!1 = !{i1 false}
diff --git a/llvm/test/Verifier/branch-uniformity-profile.ll b/llvm/test/Verifier/branch-uniformity-profile.ll
new file mode 100644
index 0000000000000..29bd4c0b310ee
--- /dev/null
+++ b/llvm/test/Verifier/branch-uniformity-profile.ll
@@ -0,0 +1,52 @@
+; RUN: not llvm-as -disable-output < %s 2>&1 | FileCheck %s
+
+; CHECK: branch.uniformity.profile is only valid on conditional branches
+declare !branch.uniformity.profile !0 void @invalid_declaration()
+
+; CHECK: branch.uniformity.profile is only valid on conditional branches
+define void @invalid_function() !branch.uniformity.profile !0 {
+ ret void
+}
+
+; CHECK: branch.uniformity.profile must be an empty node
+define void @invalid_payload(i1 %condition) {
+ br i1 %condition, label %then, label %else, !branch.uniformity.profile !1
+then:
+ ret void
+else:
+ ret void
+}
+
+; CHECK: branch.uniformity.profile is only valid on conditional branches
+define void @invalid_unconditional_branch() {
+ br label %exit, !branch.uniformity.profile !0
+exit:
+ ret void
+}
+
+; CHECK: branch.uniformity.profile is only valid on conditional branches
+define i32 @invalid_select(i1 %condition) {
+ %value = select i1 %condition, i32 1, i32 2, !branch.uniformity.profile !0
+ ret i32 %value
+}
+
+; CHECK: branch.uniformity.profile is only valid on conditional branches
+define void @invalid_return() {
+ ret void, !branch.uniformity.profile !0
+}
+
+; CHECK: branch.uniformity.profile is only valid on conditional branches
+define void @invalid_switch(i32 %value) {
+ switch i32 %value, label %exit [i32 0, label %zero], !branch.uniformity.profile !0
+zero:
+ ret void
+exit:
+ ret void
+}
+
+; CHECK: branch.uniformity.profile is only valid on conditional branches
+; CHECK-NEXT: ptr @invalid_global
+ at invalid_global = global i32 0, !branch.uniformity.profile !0
+
+!0 = !{}
+!1 = !{i1 true}
diff --git a/llvm/unittests/Transforms/Instrumentation/PGOInstrumentationTest.cpp b/llvm/unittests/Transforms/Instrumentation/PGOInstrumentationTest.cpp
index 9ccb13934cbd3..686410ac1468d 100644
--- a/llvm/unittests/Transforms/Instrumentation/PGOInstrumentationTest.cpp
+++ b/llvm/unittests/Transforms/Instrumentation/PGOInstrumentationTest.cpp
@@ -8,9 +8,14 @@
#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
#include "llvm/AsmParser/Parser.h"
+#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Module.h"
+#include "llvm/IR/Verifier.h"
#include "llvm/Passes/PassBuilder.h"
#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/ProfileData/InstrProfWriter.h"
+#include "llvm/Testing/Support/Error.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
@@ -187,4 +192,116 @@ TEST_P(PGOInstrumentationGenTest, Instrumented) {
EXPECT_FALSE(IRInstrVar->isDeclaration());
}
+struct PGOInstrumentationUseTest : Test, WithParamInterface<bool> {};
+
+INSTANTIATE_TEST_SUITE_P(ExistingMetadata, PGOInstrumentationUseTest,
+ Values(false, true));
+
+TEST_P(PGOInstrumentationUseTest, BlockUniformityMetadataUsesPresence) {
+ static constexpr StringRef Code = R"(
+ define i32 @f(i1 %cond) {
+ entry:
+ br i1 %cond, label %then, label %else
+ then:
+ ret i32 1
+ else:
+ ret i32 0
+ })";
+
+ for (unsigned UniformityMask : {0u, 1u, 2u, 3u}) {
+ SCOPED_TRACE(UniformityMask);
+ LLVMContext Context;
+ SMDiagnostic ParseError;
+ auto GenModule = parseAssemblyString(Code, ParseError, Context);
+ ASSERT_THAT(GenModule, NotNull());
+
+ PassBuilder PB;
+ LoopAnalysisManager LAM;
+ FunctionAnalysisManager FAM;
+ CGSCCAnalysisManager CGAM;
+ ModuleAnalysisManager MAM;
+ PB.registerModuleAnalyses(MAM);
+ PB.registerCGSCCAnalyses(CGAM);
+ PB.registerFunctionAnalyses(FAM);
+ PB.registerLoopAnalyses(LAM);
+ PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
+
+ ModulePassManager GenMPM;
+ GenMPM.addPass(PGOInstrumentationGen());
+ GenMPM.run(*GenModule, MAM);
+
+ Function *GenFunction = GenModule->getFunction("f");
+ ASSERT_THAT(GenFunction, NotNull());
+
+ uint64_t FunctionHash = 0;
+ unsigned NumCounters = 0;
+ std::vector<std::string> BlockNames;
+ for (Instruction &I : instructions(*GenFunction)) {
+ auto *Counter = dyn_cast<InstrProfCntrInstBase>(&I);
+ if (!Counter)
+ continue;
+
+ if (BlockNames.empty()) {
+ FunctionHash = Counter->getHash()->getZExtValue();
+ NumCounters = Counter->getNumCounters()->getZExtValue();
+ BlockNames.resize(NumCounters);
+ }
+
+ unsigned Index = Counter->getIndex()->getZExtValue();
+ ASSERT_LT(Index, NumCounters);
+ BlockNames[Index] = I.getParent()->getName().str();
+ }
+
+ ASSERT_EQ(NumCounters, 2u);
+ for (const std::string &BlockName : BlockNames)
+ ASSERT_FALSE(BlockName.empty());
+
+ std::string ProfileName = getIRPGOFuncName(*GenFunction);
+ NamedInstrProfRecord Record(ProfileName, FunctionHash,
+ std::vector<uint64_t>(NumCounters, 10));
+ Record.UniformityBits = {static_cast<uint8_t>(UniformityMask)};
+
+ InstrProfWriter Writer;
+ ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::IRInstrumentation),
+ Succeeded());
+ Writer.addRecord(std::move(Record),
+ [](Error E) { ADD_FAILURE() << toString(std::move(E)); });
+ auto Profile = Writer.writeBuffer();
+ ASSERT_THAT(Profile, NotNull());
+
+ auto FS = makeIntrusiveRefCnt<vfs::InMemoryFileSystem>();
+ ASSERT_TRUE(FS->addFile("/profile.profdata", 0, std::move(Profile)));
+
+ auto UseModule = parseAssemblyString(Code, ParseError, Context);
+ ASSERT_THAT(UseModule, NotNull());
+ Function *UseFunction = UseModule->getFunction("f");
+ ASSERT_THAT(UseFunction, NotNull());
+ if (GetParam()) {
+ MDNode *UniformMD = MDNode::get(Context, {});
+ for (BasicBlock &BB : *UseFunction)
+ BB.getTerminator()->setMetadata(
+ LLVMContext::MD_block_uniformity_profile, UniformMD);
+ }
+ ModulePassManager UseMPM;
+ UseMPM.addPass(PGOInstrumentationUse("/profile.profdata", "", false, FS));
+ UseMPM.run(*UseModule, MAM);
+ EXPECT_FALSE(verifyModule(*UseModule, &errs()));
+
+ for (unsigned I = 0; I < NumCounters; ++I) {
+ BasicBlock *BB = nullptr;
+ for (BasicBlock &Candidate : *UseFunction)
+ if (Candidate.getName() == BlockNames[I])
+ BB = &Candidate;
+ ASSERT_THAT(BB, NotNull());
+
+ MDNode *MD = BB->getTerminator()->getMetadata(
+ LLVMContext::MD_block_uniformity_profile);
+ bool IsUniform = (UniformityMask & (1u << I)) != 0;
+ EXPECT_EQ(MD != nullptr, IsUniform);
+ if (MD)
+ EXPECT_EQ(MD->getNumOperands(), 0u);
+ }
+ }
+}
+
} // end anonymous namespace
>From 06dd54cdaa44fed2742a513aae5d5de5bfabf3d6 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Fri, 11 Sep 2026 09:34:15 -0400
Subject: [PATCH 2/2] [IR] Align uniformity metadata with RFC v2
---
llvm/docs/LangRef.md | 55 ++++++++++++-------
llvm/include/llvm/IR/FixedMetadataKinds.def | 1 +
llvm/lib/IR/Verifier.cpp | 19 ++++---
.../block-uniformity-profile-metadata.ll | 4 +-
.../test/Verifier/block-uniformity-profile.ll | 15 ++---
llvm/test/Verifier/uniformity-profile.ll | 27 +++++++++
6 files changed, 81 insertions(+), 40 deletions(-)
create mode 100644 llvm/test/Verifier/uniformity-profile.ll
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index ba562c88a7c22..b66146e9b3a8a 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -7897,21 +7897,38 @@ section is not marked as readable or writable and it uses the section flag
!0 = !{}
```
+(md_uniformity_profile)=
+
+#### '`uniformity.profile`' Metadata
+
+`uniformity.profile` metadata indicates that uniformity profile data was
+loaded for a function. It may be attached only to a function definition and
+must be an empty metadata node.
+
+This marker lets an optimization distinguish a function without uniformity
+profile data from one whose individual blocks or branches are not known to be
+uniform. Missing block or branch metadata may mean that the control flow was
+divergent, was not observed, or lacked enough profile information.
+
+```llvm
+define void @example() !uniformity.profile !0 {
+ ret void
+}
+
+!0 = !{}
+```
+
(md_block_uniformity_profile)=
#### '`block.uniformity.profile`' Metadata
`block.uniformity.profile` metadata records observed SIMT execution uniformity
-from an instrumentation profile. It may be attached to a function definition
-or a terminator instruction and must be an empty metadata node.
-
-On a function definition, this metadata indicates that a uniformity profile
-was loaded. On a terminator, it indicates that the profile classified the
-basic block as usually executing with all lanes active. This classification
-may tolerate some divergent executions in the profile. A function attachment
-without any annotated terminators can represent a profile in which no instrumented block was
-uniform. An unannotated block is not known to be uniform; it may be divergent
-or lack a uniformity observation.
+from an instrumentation profile. It may be attached only to a terminator
+instruction and must be an empty metadata node. It indicates that the profile
+classified the containing basic block as usually executing with all lanes
+active. This classification may tolerate some divergent executions in the
+profile. An unannotated block is not known to be uniform; it may be divergent,
+unobserved, or lack enough profile information.
Block uniformity does not describe the uniformity of a branch decision. For
example, all lanes may reach a block and then take different successors. See
@@ -7923,9 +7940,9 @@ guarantee uniform execution on other inputs and must not be used to justify
transformations that require uniformity for correctness.
```llvm
-define void @example(i1 %condition) !block.uniformity.profile !0 {
+define void @example(i1 %condition) !uniformity.profile !0 {
entry:
- br i1 %condition, label %then, label %else, !block.uniformity.profile !0
+ br i1 %condition, label %then, label %else, !block.uniformity.profile !0, !branch.uniformity.profile !0
then:
ret void
else:
@@ -7939,15 +7956,15 @@ else:
#### '`branch.uniformity.profile`' Metadata
-`branch.uniformity.profile` metadata records that a profile classified a
-conditional branch as usually uniform across lanes in a SIMT execution group.
-It may be attached only to a conditional branch instruction and must be an
-empty metadata node. A uniform decision means the lanes choose the same
-successor on that execution; the chosen successor can vary between executions.
-The profile classification may tolerate some divergent executions.
+`branch.uniformity.profile` metadata records a profile-derived hint that every
+instrumented successor used to classify a conditional branch usually executed
+with all lanes active. It may be attached only to a conditional branch
+instruction and must be an empty metadata node. The profiler does not directly
+observe every branch decision or whether only the currently active lanes chose
+the same successor.
An unannotated branch is not known to be uniform. Optimizations can use the
-function-level {ref}`block.uniformity.profile <md_block_uniformity_profile>`
+function-level {ref}`uniformity.profile <md_uniformity_profile>`
attachment to distinguish a function with uniformity profile information from
one without it.
diff --git a/llvm/include/llvm/IR/FixedMetadataKinds.def b/llvm/include/llvm/IR/FixedMetadataKinds.def
index 76e862eae3aee..54fbc1a3065be 100644
--- a/llvm/include/llvm/IR/FixedMetadataKinds.def
+++ b/llvm/include/llvm/IR/FixedMetadataKinds.def
@@ -69,3 +69,4 @@ LLVM_FIXED_MD_KIND(MD_callgraph, "callgraph", 54)
LLVM_FIXED_MD_KIND(MD_metadata_section_kind, "metadata_section_kind", 55)
LLVM_FIXED_MD_KIND(MD_branch_uniformity_profile, "branch.uniformity.profile",
56)
+LLVM_FIXED_MD_KIND(MD_uniformity_profile, "uniformity.profile", 57)
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 97560b4af30cb..751d2b1c941db 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -586,11 +586,11 @@ void Verifier::visitGlobalValue(const GlobalValue &GV) {
Check(!GO->hasMetadata(LLVMContext::MD_branch_uniformity_profile),
"branch.uniformity.profile is only valid on conditional branches",
GO);
- if (GO->hasMetadata(LLVMContext::MD_block_uniformity_profile))
+ Check(!GO->hasMetadata(LLVMContext::MD_block_uniformity_profile),
+ "block.uniformity.profile is only valid on terminators", GO);
+ if (GO->hasMetadata(LLVMContext::MD_uniformity_profile))
Check(isa<Function>(GO) && !GO->isDeclaration(),
- "block.uniformity.profile is only valid on function definitions "
- "and terminators",
- GO);
+ "uniformity.profile is only valid on function definitions", GO);
if (const MDNode *Associated =
GO->getMetadata(LLVMContext::MD_associated)) {
@@ -2769,9 +2769,9 @@ void Verifier::verifyUnknownProfileMetadata(MDNode *MD) {
void Verifier::verifyFunctionMetadata(
ArrayRef<std::pair<unsigned, MDNode *>> MDs) {
for (const auto &Pair : MDs) {
- if (Pair.first == LLVMContext::MD_block_uniformity_profile)
+ if (Pair.first == LLVMContext::MD_uniformity_profile)
Check(Pair.second->getNumOperands() == 0,
- "block.uniformity.profile must be an empty node", Pair.second);
+ "uniformity.profile must be an empty node", Pair.second);
if (Pair.first == LLVMContext::MD_prof) {
MDNode *MD = Pair.second;
Check(MD->getNumOperands() >= 2,
@@ -6061,13 +6061,14 @@ void Verifier::visitInstruction(Instruction &I) {
if (MDNode *MD = I.getMetadata(LLVMContext::MD_block_uniformity_profile)) {
Check(I.isTerminator(),
- "block.uniformity.profile is only valid on function definitions "
- "and terminators",
- &I);
+ "block.uniformity.profile is only valid on terminators", &I);
Check(MD->getNumOperands() == 0,
"block.uniformity.profile must be an empty node", &I, MD);
}
+ Check(!I.getMetadata(LLVMContext::MD_uniformity_profile),
+ "uniformity.profile is only valid on function definitions", &I);
+
if (MDNode *MD = I.getMetadata(LLVMContext::MD_branch_uniformity_profile)) {
Check(isa<CondBrInst>(I),
"branch.uniformity.profile is only valid on conditional branches",
diff --git a/llvm/test/Bitcode/block-uniformity-profile-metadata.ll b/llvm/test/Bitcode/block-uniformity-profile-metadata.ll
index a98f1cebda69d..bcdd48bf2b505 100644
--- a/llvm/test/Bitcode/block-uniformity-profile-metadata.ll
+++ b/llvm/test/Bitcode/block-uniformity-profile-metadata.ll
@@ -1,8 +1,8 @@
; RUN: llvm-as < %s | llvm-dis | FileCheck %s
; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
-define void @branch_metadata(i1 %cond) !block.uniformity.profile !0 {
-; CHECK-LABEL: define void @branch_metadata(i1 %cond) !block.uniformity.profile !0 {
+define void @branch_metadata(i1 %cond) !uniformity.profile !0 {
+; CHECK-LABEL: define void @branch_metadata(i1 %cond) !uniformity.profile !0 {
entry:
br i1 %cond, label %uniform, label %divergent, !block.uniformity.profile !0, !branch.uniformity.profile !0
; CHECK: br i1 %cond, label %uniform, label %divergent, !block.uniformity.profile !0, !branch.uniformity.profile !0
diff --git a/llvm/test/Verifier/block-uniformity-profile.ll b/llvm/test/Verifier/block-uniformity-profile.ll
index 7a8f59fc5dcf9..7084c2196d5d9 100644
--- a/llvm/test/Verifier/block-uniformity-profile.ll
+++ b/llvm/test/Verifier/block-uniformity-profile.ll
@@ -1,15 +1,10 @@
; RUN: not llvm-as -disable-output < %s 2>&1 | FileCheck %s
-; CHECK: block.uniformity.profile is only valid on function definitions and terminators
+; CHECK: block.uniformity.profile is only valid on terminators
declare !block.uniformity.profile !0 void @invalid_declaration()
-; CHECK: block.uniformity.profile must be an empty node
-define void @invalid_function_payload() !block.uniformity.profile !1 {
- ret void
-}
-
-; CHECK: block.uniformity.profile must be an empty node
-define void @invalid_duplicate_payload() !block.uniformity.profile !1 !block.uniformity.profile !0 {
+; CHECK: block.uniformity.profile is only valid on terminators
+define void @invalid_function() !block.uniformity.profile !0 {
ret void
}
@@ -18,13 +13,13 @@ define void @invalid_terminator_payload() {
ret void, !block.uniformity.profile !1
}
-; CHECK: block.uniformity.profile is only valid on function definitions and terminators
+; CHECK: block.uniformity.profile is only valid on terminators
define i32 @invalid_instruction(i32 %value) {
%sum = add i32 %value, 1, !block.uniformity.profile !0
ret i32 %sum
}
-; CHECK: block.uniformity.profile is only valid on function definitions and terminators
+; CHECK: block.uniformity.profile is only valid on terminators
; CHECK-NEXT: ptr @invalid_global
@invalid_global = global i32 0, !block.uniformity.profile !0
diff --git a/llvm/test/Verifier/uniformity-profile.ll b/llvm/test/Verifier/uniformity-profile.ll
new file mode 100644
index 0000000000000..e30e331c40ce6
--- /dev/null
+++ b/llvm/test/Verifier/uniformity-profile.ll
@@ -0,0 +1,27 @@
+; RUN: not llvm-as -disable-output < %s 2>&1 | FileCheck %s
+
+; CHECK: uniformity.profile is only valid on function definitions
+declare !uniformity.profile !0 void @invalid_declaration()
+
+; CHECK: uniformity.profile must be an empty node
+define void @invalid_function_payload() !uniformity.profile !1 {
+ ret void
+}
+
+; CHECK: uniformity.profile must be an empty node
+define void @invalid_duplicate_payload() !uniformity.profile !1 !uniformity.profile !0 {
+ ret void
+}
+
+; CHECK: uniformity.profile is only valid on function definitions
+define i32 @invalid_instruction(i32 %value) {
+ %sum = add i32 %value, 1, !uniformity.profile !0
+ ret i32 %sum
+}
+
+; CHECK: uniformity.profile is only valid on function definitions
+; CHECK-NEXT: ptr @invalid_global
+ at invalid_global = global i32 0, !uniformity.profile !0
+
+!0 = !{}
+!1 = !{i1 false}
More information about the llvm-commits
mailing list