[llvm] [Uniformity] Implement per-output machine uniformity analysis (PR #179275)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Feb 2 08:26:53 PST 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-analysis
Author: Pankaj Dwivedi (PankajDwivedi-25)
<details>
<summary>Changes</summary>
Machine uniformity analysis currently treats uniformity as a `per-instruction` property. However, MIR instructions can produce `multiple outputs` where some may be uniform and others divergent. This patch changes the analysis to track uniformity at the per-output level.
---
Full diff: https://github.com/llvm/llvm-project/pull/179275.diff
8 Files Affected:
- (modified) llvm/include/llvm/ADT/GenericUniformityImpl.h (+16-5)
- (modified) llvm/include/llvm/CodeGen/TargetInstrInfo.h (+4-3)
- (modified) llvm/lib/Analysis/UniformityAnalysis.cpp (+1-1)
- (modified) llvm/lib/CodeGen/MachineUniformityAnalysis.cpp (+77-15)
- (modified) llvm/lib/Target/AMDGPU/SIInstrInfo.cpp (+27-11)
- (modified) llvm/lib/Target/AMDGPU/SIInstrInfo.h (+4-4)
- (modified) llvm/test/Analysis/UniformityAnalysis/AMDGPU/MIR/hidden-diverge-gmir.mir (+4-2)
- (added) llvm/test/Analysis/UniformityAnalysis/AMDGPU/MIR/per-output-uniformity.mir (+78)
``````````diff
diff --git a/llvm/include/llvm/ADT/GenericUniformityImpl.h b/llvm/include/llvm/ADT/GenericUniformityImpl.h
index 2db76a1ad9b13..55af9be64883e 100644
--- a/llvm/include/llvm/ADT/GenericUniformityImpl.h
+++ b/llvm/include/llvm/ADT/GenericUniformityImpl.h
@@ -357,8 +357,8 @@ template <typename ContextT> class GenericUniformityAnalysisImpl {
const FunctionT &getFunction() const { return F; }
- /// \brief Mark \p UniVal as a value that is always uniform.
- void addUniformOverride(const InstructionT &Instr);
+ /// \brief Mark \p V as a value that is always uniform.
+ void addUniformOverride(ConstValueRefT V);
/// \brief Examine \p I for divergent outputs and add to the worklist.
void markDivergent(const InstructionT &I);
@@ -382,6 +382,9 @@ template <typename ContextT> class GenericUniformityAnalysisImpl {
/// operands
bool isAlwaysUniform(const InstructionT &Instr) const;
+ /// \brief Whether \p V is always uniform (per-value check for MIR).
+ bool isAlwaysUniform(ConstValueRefT V) const;
+
bool hasDivergentDefs(const InstructionT &I) const;
bool isDivergent(const InstructionT &I) const {
@@ -440,7 +443,7 @@ template <typename ContextT> class GenericUniformityAnalysisImpl {
SyncDependenceAnalysisT SDA;
// Set of known-uniform values.
- SmallPtrSet<const InstructionT *, 32> UniformOverrides;
+ DenseSet<ConstValueRefT> UniformOverrides;
/// \brief Mark all nodes in \p JoinBlock as divergent and push them on
/// the worklist.
@@ -801,6 +804,8 @@ void GenericUniformityAnalysisImpl<ContextT>::markDivergent(
template <typename ContextT>
bool GenericUniformityAnalysisImpl<ContextT>::markDivergent(
ConstValueRefT Val) {
+ if (isAlwaysUniform(Val))
+ return false;
if (DivergentValues.insert(Val).second) {
LLVM_DEBUG(dbgs() << "marked divergent: " << Context.print(Val) << "\n");
return true;
@@ -810,8 +815,8 @@ bool GenericUniformityAnalysisImpl<ContextT>::markDivergent(
template <typename ContextT>
void GenericUniformityAnalysisImpl<ContextT>::addUniformOverride(
- const InstructionT &Instr) {
- UniformOverrides.insert(&Instr);
+ ConstValueRefT V) {
+ UniformOverrides.insert(V);
}
// Mark as divergent all external uses of values defined in \p DefCycle.
@@ -1143,6 +1148,12 @@ bool GenericUniformityAnalysisImpl<ContextT>::isAlwaysUniform(
return UniformOverrides.contains(&Instr);
}
+template <typename ContextT>
+bool GenericUniformityAnalysisImpl<ContextT>::isAlwaysUniform(
+ ConstValueRefT V) const {
+ return UniformOverrides.contains(V);
+}
+
template <typename ContextT>
GenericUniformityInfo<ContextT>::GenericUniformityInfo(
const DominatorTreeT &DT, const CycleInfoT &CI,
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index 91fddce7e7e47..7587e9074870b 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -2367,9 +2367,10 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
llvm_unreachable("impossible call instruction");
}
- /// Return the uniformity behavior of the given instruction.
- virtual InstructionUniformity
- getInstructionUniformity(const MachineInstr &MI) const {
+ /// Return the uniformity behavior of the given instruction's output.
+ /// \p DefIdx specifies which output to query (for multi-output instructions).
+ virtual InstructionUniformity getDefUniformity(const MachineInstr &MI,
+ unsigned DefIdx = 0) const {
return InstructionUniformity::Default;
}
diff --git a/llvm/lib/Analysis/UniformityAnalysis.cpp b/llvm/lib/Analysis/UniformityAnalysis.cpp
index b56534935d7c2..3f021584ea901 100644
--- a/llvm/lib/Analysis/UniformityAnalysis.cpp
+++ b/llvm/lib/Analysis/UniformityAnalysis.cpp
@@ -34,7 +34,7 @@ template <> void llvm::GenericUniformityAnalysisImpl<SSAContext>::initialize() {
InstructionUniformity IU = TTI->getInstructionUniformity(&I);
switch (IU) {
case InstructionUniformity::AlwaysUniform:
- addUniformOverride(I);
+ addUniformOverride(&I);
continue;
case InstructionUniformity::NeverUniform:
markDivergent(I);
diff --git a/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp b/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
index dbadb67e1e6d2..6947c12d3eb1e 100644
--- a/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
+++ b/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
@@ -46,24 +46,86 @@ bool llvm::GenericUniformityAnalysisImpl<MachineSSAContext>::markDefsDivergent(
return insertedDivergent;
}
+template <>
+bool llvm::GenericUniformityAnalysisImpl<MachineSSAContext>::isAlwaysUniform(
+ const MachineInstr &Instr) const {
+ // For MIR, an instruction is "always uniform" only if it has at least one
+ // virtual register def AND all those defs are in UniformOverrides.
+ // Instructions with no virtual register defs (e.g., terminators like
+ // G_BRCOND, G_BR) return false to ensure they can be properly processed
+ // for divergence during propagation.
+ bool HasVirtualDef = false;
+ for (const MachineOperand &Op : Instr.all_defs()) {
+ if (!Op.getReg().isVirtual())
+ continue;
+ HasVirtualDef = true;
+ if (!UniformOverrides.contains(Op.getReg()))
+ return false;
+ }
+ // Only return true if we found at least one virtual def and all were uniform
+ return HasVirtualDef;
+}
+
template <>
void llvm::GenericUniformityAnalysisImpl<MachineSSAContext>::initialize() {
- const auto &InstrInfo = *F.getSubtarget().getInstrInfo();
-
- for (const MachineBasicBlock &block : F) {
- for (const MachineInstr &instr : block) {
- auto uniformity = InstrInfo.getInstructionUniformity(instr);
-
- switch (uniformity) {
- case InstructionUniformity::AlwaysUniform:
- addUniformOverride(instr);
- break;
- case InstructionUniformity::NeverUniform:
- markDivergent(instr);
- break;
- case InstructionUniformity::Default:
- break;
+ const TargetInstrInfo &InstrInfo = *F.getSubtarget().getInstrInfo();
+ const MachineRegisterInfo &MRI = F.getRegInfo();
+ const RegisterBankInfo &RBI = *F.getSubtarget().getRegBankInfo();
+ const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
+
+ for (const MachineBasicBlock &Block : F) {
+ for (const MachineInstr &Instr : Block) {
+ // Terminators are handled separately because:
+ // 1. Many terminators (G_BRCOND, G_BR) have no def operands, so the
+ // per-def loop below would skip them entirely.
+ // 2. Divergent terminators mark the BLOCK as divergent
+ // (DivergentTermBlocks),
+ // not individual values, which is different from regular instructions.
+ // For terminators like SI_IF/SI_ELSE, getDefUniformity() checks the
+ // isNeverUniform() flag, so DefIdx is not relevant here.
+ if (Instr.isTerminator()) {
+ InstructionUniformity Uniformity = InstrInfo.getDefUniformity(Instr);
+ if (Uniformity == InstructionUniformity::NeverUniform) {
+ if (DivergentTermBlocks.insert(Instr.getParent()).second) {
+ Worklist.push_back(&Instr);
+ }
+ }
+ continue;
+ }
+
+ // Query uniformity for each def operand separately.
+ unsigned DefIdx = 0;
+ bool HasDivergentDef = false;
+ for (const MachineOperand &Op : Instr.all_defs()) {
+ if (!Op.getReg().isVirtual()) {
+ DefIdx++;
+ continue;
+ }
+
+ InstructionUniformity Uniformity =
+ InstrInfo.getDefUniformity(Instr, DefIdx);
+
+ switch (Uniformity) {
+ case InstructionUniformity::AlwaysUniform:
+ addUniformOverride(Op.getReg());
+ break;
+ case InstructionUniformity::NeverUniform:
+ // Skip registers that are inherently uniform (e.g., SGPRs on AMDGPU)
+ // even if the instruction is marked as NeverUniform.
+ if (!TRI.isUniformReg(MRI, RBI, Op.getReg())) {
+ if (markDivergent(Op.getReg()))
+ HasDivergentDef = true;
+ }
+ break;
+ case InstructionUniformity::Default:
+ break;
+ }
+ DefIdx++;
}
+ // If any def was marked divergent, add the instruction to worklist
+ // for divergence propagation to users.
+ if (HasDivergentDef)
+ Worklist.push_back(&Instr);
}
}
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 09efba485f6f8..0b775d8232603 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -10614,7 +10614,8 @@ SIInstrInfo::getCalleeOperand(const MachineInstr &MI) const {
}
InstructionUniformity
-SIInstrInfo::getGenericInstructionUniformity(const MachineInstr &MI) const {
+SIInstrInfo::getGenericDefUniformity(const MachineInstr &MI,
+ unsigned DefIdx) const {
const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
unsigned Opcode = MI.getOpcode();
@@ -10639,19 +10640,34 @@ SIInstrInfo::getGenericInstructionUniformity(const MachineInstr &MI) const {
if (Opcode == TargetOpcode::G_ADDRSPACE_CAST)
return HandleAddrSpaceCast(MI);
- if (auto *GI = dyn_cast<GIntrinsic>(&MI)) {
- auto IID = GI->getIntrinsicID();
- if (AMDGPU::isIntrinsicSourceOfDivergence(IID))
- return InstructionUniformity::NeverUniform;
+ if (const GIntrinsic *GI = dyn_cast<GIntrinsic>(&MI)) {
+ Intrinsic::ID IID = GI->getIntrinsicID();
+ if (AMDGPU::isIntrinsicSourceOfDivergence(IID)) {
+ // Some intrinsics produce multiple outputs with mixed uniformity.
+ // For these, we need to check DefIdx to determine which output is being
+ // queried and return the appropriate uniformity.
+ switch (IID) {
+ case Intrinsic::amdgcn_if:
+ case Intrinsic::amdgcn_else:
+ // These intrinsics produce two outputs:
+ // DefIdx=0: "Exec mask not zero" flag (i1) - inherits divergence from
+ // the input condition to ensure proper divergence
+ // propagation.
+ // DefIdx=1: Saved exec mask (i64) - always uniform as all active
+ // lanes see the same mask value.
+ return DefIdx == 1 ? InstructionUniformity::AlwaysUniform
+ : InstructionUniformity::Default;
+ default:
+ return InstructionUniformity::NeverUniform;
+ }
+ }
if (AMDGPU::isIntrinsicAlwaysUniform(IID))
return InstructionUniformity::AlwaysUniform;
switch (IID) {
case Intrinsic::amdgcn_addrspacecast_nonnull:
return HandleAddrSpaceCast(MI);
- case Intrinsic::amdgcn_if:
- case Intrinsic::amdgcn_else:
- // FIXME: Uniform if second result
+ default:
break;
}
@@ -10694,8 +10710,8 @@ const MIRFormatter *SIInstrInfo::getMIRFormatter() const {
return Formatter.get();
}
-InstructionUniformity
-SIInstrInfo::getInstructionUniformity(const MachineInstr &MI) const {
+InstructionUniformity SIInstrInfo::getDefUniformity(const MachineInstr &MI,
+ unsigned DefIdx) const {
if (isNeverUniform(MI))
return InstructionUniformity::NeverUniform;
@@ -10719,7 +10735,7 @@ SIInstrInfo::getInstructionUniformity(const MachineInstr &MI) const {
// GMIR handling
if (MI.isPreISelOpcode())
- return SIInstrInfo::getGenericInstructionUniformity(MI);
+ return SIInstrInfo::getGenericDefUniformity(MI, DefIdx);
// Atomics are divergent because they are executed sequentially: when an
// atomic operation refers to the same address in each thread, then each
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 05cf804d08ffc..5ab60267fd9ab 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1667,11 +1667,11 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
const MachineOperand &getCalleeOperand(const MachineInstr &MI) const override;
- InstructionUniformity
- getInstructionUniformity(const MachineInstr &MI) const final;
+ InstructionUniformity getDefUniformity(const MachineInstr &MI,
+ unsigned DefIdx = 0) const final;
- InstructionUniformity
- getGenericInstructionUniformity(const MachineInstr &MI) const;
+ InstructionUniformity getGenericDefUniformity(const MachineInstr &MI,
+ unsigned DefIdx = 0) const;
const MIRFormatter *getMIRFormatter() const override;
diff --git a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/MIR/hidden-diverge-gmir.mir b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/MIR/hidden-diverge-gmir.mir
index 27c53815feb06..e0d23d4af85dd 100644
--- a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/MIR/hidden-diverge-gmir.mir
+++ b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/MIR/hidden-diverge-gmir.mir
@@ -5,7 +5,8 @@
# CHECK: DIVERGENT: %{{[0-9]*}}: %{{[0-9]*}}:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.workitem.id.x)
# CHECK: DIVERGENT: %{{[0-9]*}}: %{{[0-9]*}}:_(s1) = G_ICMP intpred(slt)
# CHECK: DIVERGENT: %{{[0-9]*}}: %{{[0-9]*}}:_(s1) = G_XOR %{{[0-9]*}}:_, %{{[0-9]*}}:_
-# CHECK: DIVERGENT: %{{[0-9]*}}: %{{[0-9]*}}:_(s1), %{{[0-9]*}}:_(s64) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.if)
+# The first output (i1) of amdgcn.if inherits divergence from input.
+# The second output (exec mask, i64) is always uniform.
# CHECK: DIVERGENT: %{{[0-9]*}}: %{{[0-9]*}}:_(s1), %{{[0-9]*}}:_(s64) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.if)
# CHECK: DIVERGENT: G_BRCOND %{{[0-9]*}}:_(s1), %bb.1
# CHECK: DIVERGENT: G_BR %bb.2
@@ -13,7 +14,8 @@
# CHECK-LABEL: BLOCK bb.2
# CHECK-NOT: DIVERGENT: %{{[0-9]*}}: %{{[0-9]*}}:_(s32) = G_PHI %{{[0-9]*}}:_(s32), %bb.1, %{{[0-9]*}}:_(s32), %bb.0
# CHECK: DIVERGENT: %{{[0-9]*}}: %{{[0-9]*}}:_(s1) = G_PHI %{{[0-9]*}}:_(s1), %bb.1, %{{[0-9]*}}:_(s1), %bb.0
-# CHECK: DIVERGENT: %{{[0-9]*}}: %{{[0-9]*}}:_(s1), %{{[0-9]*}}:_(s64) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.if)
+# The first output (i1) of amdgcn.if inherits divergence from input.
+# The second output (exec mask, i64) is always uniform.
# CHECK: DIVERGENT: %{{[0-9]*}}: %{{[0-9]*}}:_(s1), %{{[0-9]*}}:_(s64) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.if)
# CHECK: DIVERGENT: G_BRCOND %{{[0-9]*}}:_(s1), %bb.3
# CHECK: DIVERGENT: G_BR %bb.4
diff --git a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/MIR/per-output-uniformity.mir b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/MIR/per-output-uniformity.mir
new file mode 100644
index 0000000000000..45cc2058d00d4
--- /dev/null
+++ b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/MIR/per-output-uniformity.mir
@@ -0,0 +1,78 @@
+# RUN: llc -mtriple=amdgcn-- -mcpu=gfx900 -run-pass=print-machine-uniformity -o - %s 2>&1 | FileCheck %s
+# RUN: llc -mtriple=amdgcn-- -mcpu=gfx900 -passes='print<machine-uniformity>' -filetype=null %s 2>&1 | FileCheck %s
+
+# Test per-output uniformity analysis for amdgcn.if and amdgcn.else intrinsics.
+# These intrinsics produce two outputs:
+# - First result (i1): Inherits divergence from the input condition.
+# - Second result (i64): Saved exec mask - always uniform.
+
+# Test amdgcn.if with UNIFORM input - both outputs should be uniform
+---
+name: amdgcn_if_uniform_input
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: MachineUniformityInfo for function: @amdgcn_if_uniform_input
+ ; With uniform input, both outputs are uniform
+ ; CHECK: ALL VALUES UNIFORM
+ %0:_(s1) = G_IMPLICIT_DEF
+ %1:_(s1), %2:_(s64) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.if), %0:_(s1)
+ S_ENDPGM 0
+...
+
+# Test amdgcn.if with DIVERGENT input - first output divergent, second uniform
+---
+name: amdgcn_if_divergent_input
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: MachineUniformityInfo for function: @amdgcn_if_divergent_input
+ ; First output (%3, i1) inherits divergence from the divergent input
+ ; CHECK: DIVERGENT: %3
+ ; Second output (%4, exec mask) is always uniform regardless of input
+ ; CHECK-NOT: DIVERGENT: %4
+ %0:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.workitem.id.x)
+ %1:_(s32) = G_CONSTANT i32 16
+ %2:_(s1) = G_ICMP intpred(slt), %0:_(s32), %1:_
+ %3:_(s1), %4:_(s64) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.if), %2:_(s1)
+ S_ENDPGM 0
+...
+
+# Test amdgcn.else with UNIFORM input - both outputs should be uniform
+---
+name: amdgcn_else_uniform_input
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: MachineUniformityInfo for function: @amdgcn_else_uniform_input
+ ; With uniform input, both outputs are uniform
+ ; CHECK: ALL VALUES UNIFORM
+ %0:_(s64) = G_IMPLICIT_DEF
+ %1:_(s1), %2:_(s64) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.else), %0:_(s64)
+ S_ENDPGM 0
+...
+
+# Test amdgcn.else with DIVERGENT input - first output divergent, second uniform
+---
+name: amdgcn_else_divergent_input
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: MachineUniformityInfo for function: @amdgcn_else_divergent_input
+ ; First output (%2, i1) inherits divergence from the divergent input
+ ; CHECK: DIVERGENT: %2
+ ; Second output (%3, exec mask) is always uniform regardless of input
+ ; CHECK-NOT: DIVERGENT: %3
+ %0:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.workitem.id.x)
+ %1:_(s64) = G_ZEXT %0:_(s32)
+ %2:_(s1), %3:_(s64) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.else), %1:_(s64)
+ S_ENDPGM 0
+...
``````````
</details>
https://github.com/llvm/llvm-project/pull/179275
More information about the llvm-commits
mailing list