[llvm] [AMDGPU] using divergent/uniform information in ISel of zext (PR #174539)
Zeng Wu via llvm-commits
llvm-commits at lists.llvm.org
Sat Apr 18 22:25:43 PDT 2026
https://github.com/zwu-2025 updated https://github.com/llvm/llvm-project/pull/174539
>From b20ed163ed103c6d02036bd64af37ab5c610d7c2 Mon Sep 17 00:00:00 2001
From: zwu-2025 <Zeng.Wu2 at amd.com>
Date: Mon, 23 Mar 2026 17:49:18 +0000
Subject: [PATCH 1/3] changes decision order of UseRC and use
common-sub-regclass
---
.../lib/CodeGen/SelectionDAG/InstrEmitter.cpp | 21 +++++++++++++------
1 file changed, 15 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 322766dee5aa9..20c02317019c6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -24,9 +24,11 @@
#include "llvm/CodeGen/StackMaps.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetLowering.h"
+#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/PseudoProbe.h"
+#include "llvm/MC/MCInstrDesc.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Target/TargetMachine.h"
using namespace llvm;
@@ -103,12 +105,7 @@ void InstrEmitter::EmitCopyFromReg(SDValue Op, bool IsClone, Register SrcReg,
bool MatchReg = true;
MVT VT = Op.getSimpleValueType();
-
- // FIXME: The Untyped check is a workaround for SystemZ i128 inline assembly
- // using i128, when it should probably be using v2i64.
- const TargetRegisterClass *UseRC =
- VT == MVT::Untyped ? nullptr : TLI->getRegClassFor(VT, Op->isDivergent());
-
+ const TargetRegisterClass *UseRC = nullptr;
for (SDNode *User : Op->users()) {
bool Match = true;
if (User->getOpcode() == ISD::CopyToReg && User->getOperand(2) == Op) {
@@ -132,6 +129,7 @@ void InstrEmitter::EmitCopyFromReg(SDValue Op, bool IsClone, Register SrcReg,
RC = TRI->getAllocatableClass(
TII->getRegClass(II, i + II.getNumDefs()));
}
+
if (!UseRC)
UseRC = RC;
else if (RC) {
@@ -150,6 +148,17 @@ void InstrEmitter::EmitCopyFromReg(SDValue Op, bool IsClone, Register SrcReg,
break;
}
+ // FIXME: The Untyped check is a workaround for SystemZ i128 inline assembly
+ // using i128, when it should probably be using v2i64.
+ const TargetRegisterClass *RegClassForVT =
+ VT == MVT::Untyped ? nullptr : TLI->getRegClassFor(VT, Op->isDivergent());
+
+ if (UseRC == nullptr || !UseRC->isAllocatable()) {
+ UseRC = RegClassForVT;
+ } else if (auto CommonSubClass = TRI->getCommonSubClass(UseRC, RegClassForVT)) {
+ UseRC = CommonSubClass;
+ }
+
const TargetRegisterClass *SrcRC = nullptr, *DstRC = nullptr;
SrcRC = TRI->getMinimalPhysRegClass(SrcReg, VT);
>From ce4a770ba8f8b677a497bae30bb97d4b48abf087 Mon Sep 17 00:00:00 2001
From: zwu-2025 <Zeng.Wu2 at amd.com>
Date: Mon, 23 Mar 2026 21:58:21 +0000
Subject: [PATCH 2/3] Select S_AND for ZEXT Only if directly from ICMP and
UNIFORM
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 61 +++++++++++++++++--
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h | 1 +
.../llvm.amdgcn.uniform_ext_i1_to_i32.ll | 47 ++++++++++++++
3 files changed, 105 insertions(+), 4 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.uniform_ext_i1_to_i32.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index cc2058a5a1d4a..d630cf46b7dbe 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -325,10 +325,57 @@ bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const {
return false;
}
-void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
- if (!Subtarget->d16PreservesUnusedBits())
- return;
+bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
+ // For the DAG:
+ // t134: i1 = setcc t52, Constant:i32<0>, setne:ch
+ // t133: i32 = zero_extend t134
+ // After DAG selections, it is:
+ // t134: i1 = S_CMP_LG_U32 t52, t19
+ // t133: i32,i1 = S_AND_B32 t134, TargetConstant:i32<1>
+ //
+ // And the listed MIR are:
+ // S_CMP_LG_U32 killed %25:sreg_32, %26:sreg_32, implicit-def $scc
+ // %27:sreg_32 = COPY $scc
+ // %28:sreg_32 = S_AND_B32 killed %27:sreg_32, 1, implicit-def dead $scc
+ //
+ // After SI-Fix-SGPR-Copies, if wave size is 64, the fixSCCCopy will change it to
+ // S_CMP_LG_U32 killed %25:sreg_32, %26:sreg_32, implicit-def $scc
+ // %79:sreg_64_xexec = S_CSELECT_B64 -1, 0, implicit $scc
+ // %27:sreg_32 = COPY %79:sreg_64_xexec; illegal vgpr to sgpr copy
+ //
+ // This preprocess changes DAG
+ // t133: i32 = zero_extend t134 into --> S_CSELEECT_B32 -1 0
+ // if it is valid.
+ if (N->isDivergent()) return false;
+
+ // If to i64, it is probably used for lane mask, then we don't do the changes here.
+ if (N->getValueType(0) != MVT::i32) return false;
+
+ auto CondNode = N->getOperand(0);
+ if (CondNode->getOpcode() == ISD::FREEZE) {
+ CondNode = CondNode->getOperand(0);
+ }
+
+ if (CondNode->getOpcode() != ISD::SETCC)
+ return false;
+
+ if (CondNode->getOperand(0).getValueType() != MVT::i32 || CondNode->getOperand(1).getValueType() != MVT::i32)
+ return false;
+
+ SDLoc DL(N);
+ SDValue TrueValue = CurDAG->getAllOnesConstant(DL, MVT::i32, true);
+ SDValue FalseValue = CurDAG->getTargetConstant(0, DL, MVT::i32);
+ SDValue Chain = CurDAG->getEntryNode(); // Basic chain to anchor the copy
+ SDValue CopyToSCC = CurDAG->getCopyToReg(Chain, DL, AMDGPU::SCC, CondNode, SDValue());
+ SmallVector<SDValue, 3> Ops = {TrueValue, FalseValue, CopyToSCC.getValue(1)};
+ MachineSDNode *CSelectNode = CurDAG->getMachineNode(AMDGPU::S_CSELECT_B32, DL, CurDAG->getVTList(MVT::i32), Ops);
+ CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), SDValue(CSelectNode, 0));
+
+ return true;
+}
+
+void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
bool MadeChange = false;
@@ -340,8 +387,14 @@ void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
switch (N->getOpcode()) {
case ISD::BUILD_VECTOR:
// TODO: Match load d16 from shl (extload:i16), 16
- MadeChange |= matchLoadD16FromBuildVector(N);
+ if (Subtarget->d16PreservesUnusedBits())
+ MadeChange |= matchLoadD16FromBuildVector(N);
+
break;
+ case ISD::ZERO_EXTEND:
+ MadeChange |= preprocessZeroExtend(N);
+ break;
+
default:
break;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
index ffeb6dfdb3f90..9cc34cc769cf0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
@@ -67,6 +67,7 @@ class AMDGPUDAGToDAGISel : public SelectionDAGISel {
bool runOnMachineFunction(MachineFunction &MF) override;
bool matchLoadD16FromBuildVector(SDNode *N) const;
+ bool preprocessZeroExtend(SDNode *N) const;
void PreprocessISelDAG() override;
void Select(SDNode *N) override;
void PostprocessISelDAG() override;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.uniform_ext_i1_to_i32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.uniform_ext_i1_to_i32.ll
new file mode 100644
index 0000000000000..f5cc63694be9a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.uniform_ext_i1_to_i32.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck --check-prefix=GFX950 %s
+
+declare i32 @llvm.amdgcn.workitem.id.x() #0
+
+define amdgpu_kernel void @zext_i1_to_i64_uniform(ptr addrspace(1) %out, i1 %pred, i32 %a, i32 %b) #0 {
+entry:
+ ; GFX950-LABEL: zext_i1_to_i64_uniform
+ ; GFX950: s_load_dwordx2 s[0:1], s[4:5], 0x30
+ ; GFX950-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+ ; GFX950-NEXT: s_mov_b32 s4, 0
+ ; GFX950-NEXT: v_mov_b32_e32 v2, 0
+ ; GFX950-NEXT: v_mov_b32_e32 v1, s4
+ ; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+ ; GFX950-NEXT: s_cmp_eq_u32 s0, s1
+ ; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
+ ; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+ ; GFX950-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+ ; GFX950-NEXT: s_endpgm
+
+ %cmp = icmp eq i32 %a, %b
+ %tmp2 = zext i1 %cmp to i64
+ store i64 %tmp2, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @zext_i1_to_i32_uniform(ptr addrspace(1) %out, i1 %pred, i32 %a, i32 %b) #0 {
+entry:
+ ; GFX950-LABEL: zext_i1_to_i32_uniform
+ ; GFX950: s_load_dwordx2 s[0:1], s[4:5], 0x30
+ ; GFX950-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+ ; GFX950-NEXT: v_mov_b32_e32 v0, 0
+ ; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+ ; GFX950-NEXT: s_cmp_eq_u32 s0, s1
+ ; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
+ ; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
+ ; GFX950-NEXT: s_cselect_b32 s0, -1, 0
+ ; GFX950-NEXT: v_mov_b32_e32 v1, s0
+ ; GFX950-NEXT: global_store_dword v0, v1, s[2:3]
+ ; GFX950-NEXT: s_endpgm
+ %cmp = icmp eq i32 %a, %b
+ %tmp2 = zext i1 %cmp to i32
+ store i32 %tmp2, ptr addrspace(1) %out
+ ret void
+}
+
+attributes #0 = { nounwind }
>From 15bfca277795cd76d83a394eaef3a82f2ca6fec9 Mon Sep 17 00:00:00 2001
From: zwu-2025 <Zeng.Wu2 at amd.com>
Date: Sun, 19 Apr 2026 05:25:02 +0000
Subject: [PATCH 3/3] validate scc is not cross bb before and after
si-fix-sgpr-copies
---
llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp | 63 ++++++++++++++++++++++
1 file changed, 63 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
index 5994eeb54095b..991cb05934b7f 100644
--- a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
@@ -159,6 +159,8 @@ class SIFixSGPRCopies {
MachineBasicBlock *BlockToInsertTo,
MachineBasicBlock::iterator PointToInsertTo,
const DebugLoc &DL);
+
+ void validateNoCrossBlockSCC(MachineFunction &MF);
};
class SIFixSGPRCopiesLegacy : public MachineFunctionPass {
@@ -623,6 +625,8 @@ bool SIFixSGPRCopies::run(MachineFunction &MF) {
if (MF.getProperties().hasSelected())
return false;
+ validateNoCrossBlockSCC(MF);
+
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
MRI = &MF.getRegInfo();
TRI = ST.getRegisterInfo();
@@ -781,8 +785,12 @@ bool SIFixSGPRCopies::run(MachineFunction &MF) {
}
lowerVGPR2SGPRCopies(MF);
+
+ validateNoCrossBlockSCC(MF);
// Postprocessing
fixSCCCopies(MF);
+ validateNoCrossBlockSCC(MF);
+
for (auto *MI : S2VCopies) {
// Check if it is still valid
if (MI->isCopy()) {
@@ -806,6 +814,8 @@ bool SIFixSGPRCopies::run(MachineFunction &MF) {
if (MF.getTarget().getOptLevel() > CodeGenOptLevel::None && EnableM0Merge)
hoistAndMergeSGPRInits(AMDGPU::M0, *MRI, TRI, *MDT, TII);
+ validateNoCrossBlockSCC(MF);
+
SiblingPenalty.clear();
V2SCopies.clear();
SCCCopies.clear();
@@ -1012,6 +1022,10 @@ void SIFixSGPRCopies::analyzeVGPRToSGPRCopy(MachineInstr* MI) {
AnalysisWorklist.push_back(U);
}
}
+
+ for (auto I : Info.SChain) {
+ LLVM_DEBUG(dbgs() << "Chain 12345:"; I->dump(););
+ }
V2SCopies[Info.ID] = std::move(Info);
}
@@ -1055,6 +1069,55 @@ bool SIFixSGPRCopies::needToBeConvertedToVALU(V2SCopyInfo *Info) {
return Info->NeedToBeConvertedToVALU;
}
+void SIFixSGPRCopies::validateNoCrossBlockSCC(MachineFunction &MF) {
+
+ for (const MachineBasicBlock &MBB : MF) {
+ // assert(!MBB.isLiveIn(AMDGPU::SCC) && "AMDGPU::SCC is live-in to a basic
+ // block! Cross-BB SCC liveness detected.");
+
+ bool SCCDefinedInCurrentBlock = false;
+
+ for (const MachineInstr &MI : MBB) {
+ bool ReadsSCC = false;
+ bool ModifiesSCC = false;
+
+ for (const MachineOperand &MO : MI.operands()) {
+ if (MO.isRegMask() && MO.clobbersPhysReg(AMDGPU::SCC)) {
+ ModifiesSCC = true;
+ continue;
+ }
+
+ if (!MO.isReg())
+ continue;
+
+ Register Reg = MO.getReg();
+ if (!Reg || !Reg.isPhysical())
+ continue;
+
+ if (TRI->regsOverlap(Reg, AMDGPU::SCC)) {
+ if (MO.readsReg())
+ ReadsSCC = true;
+ if (MO.isDef())
+ ModifiesSCC = true;
+ }
+ }
+
+ if (ReadsSCC) {
+ if (!SCCDefinedInCurrentBlock) {
+ errs() << "Violation in MBB: " << MBB.getName() << "\n";
+ errs() << "Instruction: " << MI << "\n";
+ llvm_unreachable(
+ "SCC is read before being defined in the same basic block!");
+ }
+ }
+
+ if (ModifiesSCC) {
+ SCCDefinedInCurrentBlock = true;
+ }
+ }
+ }
+}
+
void SIFixSGPRCopies::lowerVGPR2SGPRCopies(MachineFunction &MF) {
SmallVector<unsigned, 8> LoweringWorklist;
More information about the llvm-commits
mailing list