[llvm] [Uniformity] Implement per-output machine uniformity analysis (PR #179275)
Pankaj Dwivedi via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 8 10:52:27 PDT 2026
https://github.com/PankajDwivedi-25 updated https://github.com/llvm/llvm-project/pull/179275
>From 31a492904523d24bf9f23e4c3a50b3791b2bceeb Mon Sep 17 00:00:00 2001
From: padivedi <pankajkumar.divedi at amd.com>
Date: Mon, 8 Jun 2026 23:22:06 +0530
Subject: [PATCH] [AMDGPU] Make machine uniformity analysis query uniformity
per def
---
llvm/include/llvm/ADT/GenericUniformityImpl.h | 26 +++++--
llvm/include/llvm/CodeGen/TargetInstrInfo.h | 15 +++-
llvm/lib/Analysis/UniformityAnalysis.cpp | 2 +-
.../lib/CodeGen/MachineUniformityAnalysis.cpp | 75 +++++++++++++++----
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 47 +++++++-----
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 5 +-
6 files changed, 128 insertions(+), 42 deletions(-)
diff --git a/llvm/include/llvm/ADT/GenericUniformityImpl.h b/llvm/include/llvm/ADT/GenericUniformityImpl.h
index a9a1211947492..28ef89e01a5ab 100644
--- a/llvm/include/llvm/ADT/GenericUniformityImpl.h
+++ b/llvm/include/llvm/ADT/GenericUniformityImpl.h
@@ -357,8 +357,8 @@ template <typename ContextT> class GenericUniformityAnalysisImpl {
const FunctionT &getFunction() const { return F; }
- /// \brief Mark \p UniVal as a value that is always uniform.
- void addUniformOverride(const InstructionT &Instr);
+ /// \brief Mark \p V as a value that is always uniform.
+ void addUniformOverride(ConstValueRefT V);
/// \brief Examine \p I for divergent outputs and add to the worklist.
void markDivergent(const InstructionT &I);
@@ -380,6 +380,9 @@ template <typename ContextT> class GenericUniformityAnalysisImpl {
/// operands
bool isAlwaysUniform(const InstructionT &Instr) const;
+ /// \brief Whether \p V is a value that is always uniform.
+ bool isAlwaysUniform(ConstValueRefT V) const;
+
bool hasDivergentDefs(const InstructionT &I) const;
bool isDivergent(const InstructionT &I) const {
@@ -473,8 +476,9 @@ template <typename ContextT> class GenericUniformityAnalysisImpl {
// The SDA links divergent branches to divergent control-flow joins.
SyncDependenceAnalysisT SDA;
- // Set of known-uniform values.
- SmallPtrSet<const InstructionT *, 32> UniformOverrides;
+ // Set of known-uniform values. Keyed on values (ConstValueRefT) so that
+ // individual defs of a multi-def instruction can be overridden uniform.
+ DenseSet<ConstValueRefT> UniformOverrides;
/// \brief Mark all nodes in \p JoinBlock as divergent and push them on
/// the worklist.
@@ -842,6 +846,8 @@ void GenericUniformityAnalysisImpl<ContextT>::markDivergent(
template <typename ContextT>
bool GenericUniformityAnalysisImpl<ContextT>::markDivergent(
ConstValueRefT Val) {
+ if (isAlwaysUniform(Val))
+ return false;
if (UniformValues.erase(Val)) {
LLVM_DEBUG(dbgs() << "marked divergent: " << Context.print(Val) << "\n");
return true;
@@ -851,8 +857,8 @@ bool GenericUniformityAnalysisImpl<ContextT>::markDivergent(
template <typename ContextT>
void GenericUniformityAnalysisImpl<ContextT>::addUniformOverride(
- const InstructionT &Instr) {
- UniformOverrides.insert(&Instr);
+ ConstValueRefT V) {
+ UniformOverrides.insert(V);
}
template <typename ContextT>
@@ -1182,7 +1188,13 @@ void GenericUniformityAnalysisImpl<ContextT>::recordTemporalDivergence(
template <typename ContextT>
bool GenericUniformityAnalysisImpl<ContextT>::isAlwaysUniform(
const InstructionT &Instr) const {
- return UniformOverrides.contains(&Instr);
+ return isAlwaysUniform(&Instr);
+}
+
+template <typename ContextT>
+bool GenericUniformityAnalysisImpl<ContextT>::isAlwaysUniform(
+ ConstValueRefT V) const {
+ return UniformOverrides.contains(V);
}
template <typename ContextT>
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index 03f3bf26d0608..c24fe3a24500b 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -2369,11 +2369,22 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
llvm_unreachable("impossible call instruction");
}
- /// Return the uniformity behavior of the given value.
- virtual ValueUniformity getValueUniformity(const MachineInstr &MI) const {
+ /// Return the uniformity behavior of the value defined by the \p DefIdx-th
+ /// def operand (in MachineInstr::all_defs() order) of \p MI. \p MI must not
+ /// be a terminator and \p DefIdx must refer to a virtual register def; use
+ /// isTerminatorDivergent() to query branch divergence for terminators.
+ virtual ValueUniformity getValueUniformity(const MachineInstr &MI,
+ unsigned DefIdx) const {
return ValueUniformity::Default;
}
+ /// Return true if \p MI is a terminator whose branch divergence is
+ /// independent of its operands (an unconditional source of control
+ /// divergence).
+ virtual bool isTerminatorDivergent(const MachineInstr &MI) const {
+ return false;
+ }
+
/// Returns true if the given \p MI defines a TargetIndex operand that can be
/// tracked by their offset, can have values, and can have debug info
/// associated with it. If so, sets \p Index and \p Offset of the target index
diff --git a/llvm/lib/Analysis/UniformityAnalysis.cpp b/llvm/lib/Analysis/UniformityAnalysis.cpp
index 73b6476fb7b6d..bc85287bfe138 100644
--- a/llvm/lib/Analysis/UniformityAnalysis.cpp
+++ b/llvm/lib/Analysis/UniformityAnalysis.cpp
@@ -81,7 +81,7 @@ template <> void llvm::GenericUniformityAnalysisImpl<SSAContext>::initialize() {
switch (IU) {
case ValueUniformity::AlwaysUniform:
UniformValues.insert(&I);
- addUniformOverride(I);
+ addUniformOverride(&I);
continue;
case ValueUniformity::NeverUniform:
// Skip inserting -- divergent by definition. Add to Worklist directly
diff --git a/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp b/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
index 03a046ea995ed..07188a0b11815 100644
--- a/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
+++ b/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
@@ -13,6 +13,7 @@
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/MachineSSAContext.h"
+#include "llvm/CodeGen/RegisterBankInfo.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/InitializePasses.h"
@@ -46,6 +47,25 @@ bool llvm::GenericUniformityAnalysisImpl<MachineSSAContext>::markDefsDivergent(
return InsertedDivergent;
}
+template <>
+bool llvm::GenericUniformityAnalysisImpl<MachineSSAContext>::isAlwaysUniform(
+ const MachineInstr &Instr) const {
+ // An instruction is always uniform only if it has at least one virtual
+ // register def and every virtual def has been overridden uniform. Because
+ // overrides are tracked per def, a multi-def instruction with a mix of
+ // uniform and divergent outputs is not considered always uniform.
+ bool HasVirtualDef = false;
+ for (const MachineOperand &Op : Instr.all_defs()) {
+ Register Reg = Op.getReg();
+ if (!Reg.isVirtual())
+ continue;
+ HasVirtualDef = true;
+ if (!isAlwaysUniform(Reg))
+ return false;
+ }
+ return HasVirtualDef;
+}
+
template <>
void llvm::GenericUniformityAnalysisImpl<MachineSSAContext>::initialize() {
// Pre-populate UniformValues with all register defs. Physical register defs
@@ -62,24 +82,51 @@ void llvm::GenericUniformityAnalysisImpl<MachineSSAContext>::initialize() {
}
}
- const auto &InstrInfo = *F.getSubtarget().getInstrInfo();
+ const TargetInstrInfo &InstrInfo = *F.getSubtarget().getInstrInfo();
+ const MachineRegisterInfo &MRI = F.getRegInfo();
+ const RegisterBankInfo &RBI = *F.getSubtarget().getRegBankInfo();
+ const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
for (const MachineBasicBlock &MBB : F) {
for (const MachineInstr &MI : MBB) {
- ValueUniformity VU = InstrInfo.getValueUniformity(MI);
-
- switch (VU) {
- case ValueUniformity::AlwaysUniform:
- addUniformOverride(MI);
- break;
- case ValueUniformity::NeverUniform:
- markDivergent(MI);
- break;
- case ValueUniformity::Custom:
- break;
- case ValueUniformity::Default:
- break;
+ // A terminator is a source of control divergence rather than a value;
+ // seed only the unconditionally divergent ones as divergent term blocks.
+ if (MI.isTerminator()) {
+ if (InstrInfo.isTerminatorDivergent(MI))
+ markDivergent(MI);
+ continue;
+ }
+
+ // Seed divergence per def, so an instruction with several outputs (e.g.
+ // inline asm) can mix uniform and divergent results.
+ unsigned DefIdx = 0;
+ bool HasDivergentDef = false;
+ for (const MachineOperand &Op : MI.all_defs()) {
+ Register Reg = Op.getReg();
+ if (!Reg.isVirtual()) {
+ ++DefIdx;
+ continue;
+ }
+ switch (InstrInfo.getValueUniformity(MI, DefIdx++)) {
+ case ValueUniformity::AlwaysUniform:
+ addUniformOverride(Reg);
+ break;
+ case ValueUniformity::NeverUniform:
+ // Inherently uniform registers (e.g. SGPRs) stay uniform even when
+ // the def is reported as a divergence source.
+ if (!TRI.isUniformReg(MRI, RBI, Reg))
+ HasDivergentDef |= markDivergent(Reg);
+ break;
+ case ValueUniformity::Custom:
+ break;
+ case ValueUniformity::Default:
+ break;
+ }
}
+ // Queue the instruction once for propagation if any of its defs became
+ // divergent; the value-level markDivergent() does not touch the worklist.
+ if (HasDivergentDef)
+ Worklist.push_back(&MI);
}
}
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index ef9d184555bd6..ff0ac6fb30d07 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -11003,30 +11003,40 @@ const MIRFormatter *SIInstrInfo::getMIRFormatter() const {
return Formatter.get();
}
-ValueUniformity SIInstrInfo::getValueUniformity(const MachineInstr &MI) const {
+bool SIInstrInfo::isTerminatorDivergent(const MachineInstr &MI) const {
+ assert(MI.isTerminator());
+ return isNeverUniform(MI);
+}
+
+ValueUniformity SIInstrInfo::getValueUniformity(const MachineInstr &MI,
+ unsigned DefIdx) const {
+ assert(!MI.isTerminator() &&
+ "use isTerminatorDivergent() to query terminator divergence");
+ assert(DefIdx < (unsigned)std::distance(MI.all_defs().begin(),
+ MI.all_defs().end()) &&
+ "DefIdx is out of range for this instruction's defs");
+ assert(std::next(MI.all_defs().begin(), DefIdx)->getReg().isVirtual() &&
+ "DefIdx must name a virtual register def, not a physical register");
if (isNeverUniform(MI))
return ValueUniformity::NeverUniform;
+ // Inline asm can define several registers with different reg classes, so the
+ // uniformity of each output is answered individually from its def reg class.
+ if (MI.isInlineAsm()) {
+ const MachineOperand &MO = *std::next(MI.all_defs().begin(), DefIdx);
+ const TargetRegisterClass *RC =
+ MI.getRegClassConstraint(MO.getOperandNo(), this, &RI);
+ return (!RC || !RI.isSGPRClass(RC)) ? ValueUniformity::NeverUniform
+ : ValueUniformity::Default;
+ }
+
unsigned opcode = MI.getOpcode();
if (opcode == AMDGPU::V_READLANE_B32 ||
opcode == AMDGPU::V_READFIRSTLANE_B32 ||
opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR)
return ValueUniformity::AlwaysUniform;
- // If any of defs is divergent, report as NeverUniform. isUniformReg will
- // calculate in more detail for each def from its reg class, if available.
- if (MI.isInlineAsm()) {
- for (const MachineOperand &MO : MI.operands()) {
- if (!MO.isReg() || !MO.isDef())
- continue;
- const TargetRegisterClass *RC =
- MI.getRegClassConstraint(MO.getOperandNo(), this, &RI);
- if (!RC || !RI.isSGPRClass(RC))
- return ValueUniformity::NeverUniform;
- }
- }
-
if (isCopyInstr(MI)) {
const MachineOperand &srcOp = MI.getOperand(1);
if (srcOp.isReg() && srcOp.getReg().isPhysical()) {
@@ -11071,9 +11081,12 @@ ValueUniformity SIInstrInfo::getValueUniformity(const MachineInstr &MI) const {
const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
const AMDGPURegisterBankInfo *RBI = ST.getRegBankInfo();
- // FIXME: It's conceptually broken to report this for an instruction, and not
- // a specific def operand. For inline asm in particular, there could be mixed
- // uniform and divergent results.
+ // Fallback: the result is divergent if any source operand is divergent. This
+ // gives one whole-instruction verdict (DefIdx is unused), which is exact for
+ // the single-def instructions that reach here.
+ // TODO: For a true per-def answer, scan only the sources feeding MI's DefIdx
+ // def; that input->output mapping is opcode-specific, so it would need
+ // per-opcode handling (similar to the inline-asm case above).
for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
const MachineOperand &SrcOp = MI.getOperand(I);
if (!SrcOp.isReg())
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 831aa9ebb8435..75181434374cb 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1755,10 +1755,13 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
const MachineOperand &getCalleeOperand(const MachineInstr &MI) const override;
- ValueUniformity getValueUniformity(const MachineInstr &MI) const final;
+ ValueUniformity getValueUniformity(const MachineInstr &MI,
+ unsigned DefIdx) const final;
ValueUniformity getGenericValueUniformity(const MachineInstr &MI) const;
+ bool isTerminatorDivergent(const MachineInstr &MI) const final;
+
const MIRFormatter *getMIRFormatter() const override;
static unsigned getDSShaderTypeValue(const MachineFunction &MF);
More information about the llvm-commits
mailing list