[llvm] [AMDGPU]Refactor `lowerWaveReduce` for maintainability (PR #189223)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 13 02:57:11 PDT 2026
https://github.com/easyonaadit updated https://github.com/llvm/llvm-project/pull/189223
>From 77c4b9a38f70de584ebdf491d1ac2a99f3f2b90e Mon Sep 17 00:00:00 2001
From: Aaditya <Aaditya.AlokDeshpande at amd.com>
Date: Wed, 25 Mar 2026 12:24:31 +0530
Subject: [PATCH 1/6] [AMDGPU]Refactor `lowerWaveReduce` for maintainability
The function to lower wave reduce pseudos is already quite
large ,and there are yet a few more operations to support.
Refactoring some of the code to make it more manageable.
Summary of changes:
1. Moved the expansion for `V_CNDMASK_B64_PSEUDO` to a
separate function. It's needed for 64 bit dpp operations.
2. Collapsed `getIdentityValueFor32BitWaveReduction` and
`getIdentityValueFor64BitWaveReduction` into a single
function which returns a 64 bit unsigned value.
3. Merged `isFloatingPointWaveReduceOperation` and
`is32bitWaveReduceOperation` into a single function,
`ClassifyWaveReductionOp`, to return both values.
4. Modified `getDPPOpcForWaveReduction` to also return
the `Clamp` opcode.
5. Added two lambdas: `ExtractSubRegs` and `BuildRegSequence`,
as those code blocks are repeated with little variation.
6. Moved logic for setting identity value in inactive lanes
to `BuildSetInactiveInstr`.
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 422 ++++++++++------------
1 file changed, 196 insertions(+), 226 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index d0fceb6717c38..465bf3981cd5a 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -5560,7 +5560,68 @@ static MachineBasicBlock *Expand64BitScalarArithmetic(MachineInstr &MI,
return BB;
}
-static uint32_t getIdentityValueFor32BitWaveReduction(unsigned Opc) {
+static MachineBasicBlock *Expand64BitV_CND_MASK(MachineInstr &MI,
+ MachineBasicBlock *BB) {
+ MachineFunction *MF = BB->getParent();
+ const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
+ const SIInstrInfo *TII = ST.getInstrInfo();
+ const SIRegisterInfo *TRI = ST.getRegisterInfo();
+ MachineRegisterInfo &MRI = MF->getRegInfo();
+ const DebugLoc &DL = MI.getDebugLoc();
+ Register Dst = MI.getOperand(0).getReg();
+ const MachineOperand &Src0 = MI.getOperand(1);
+ const MachineOperand &Src1 = MI.getOperand(2);
+ Register SrcCond = MI.getOperand(3).getReg();
+
+ Register DstLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ Register DstHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ const auto *CondRC = TRI->getWaveMaskRegClass();
+ Register SrcCondCopy = MRI.createVirtualRegister(CondRC);
+
+ const TargetRegisterClass *Src0RC =
+ Src0.isReg() ? MRI.getRegClass(Src0.getReg()) : &AMDGPU::VReg_64RegClass;
+ const TargetRegisterClass *Src1RC =
+ Src1.isReg() ? MRI.getRegClass(Src1.getReg()) : &AMDGPU::VReg_64RegClass;
+
+ const TargetRegisterClass *Src0SubRC =
+ TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
+ const TargetRegisterClass *Src1SubRC =
+ TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
+
+ MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(
+ MI, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC);
+ MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(
+ MI, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC);
+
+ MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(
+ MI, MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC);
+ MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
+ MI, MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC);
+
+ BuildMI(*BB, MI, DL, TII->get(AMDGPU::COPY), SrcCondCopy).addReg(SrcCond);
+ BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstLo)
+ .addImm(0)
+ .add(Src0Sub0)
+ .addImm(0)
+ .add(Src1Sub0)
+ .addReg(SrcCondCopy);
+ BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstHi)
+ .addImm(0)
+ .add(Src0Sub1)
+ .addImm(0)
+ .add(Src1Sub1)
+ .addReg(SrcCondCopy);
+
+ BuildMI(*BB, MI, DL, TII->get(AMDGPU::REG_SEQUENCE), Dst)
+ .addReg(DstLo)
+ .addImm(AMDGPU::sub0)
+ .addReg(DstHi)
+ .addImm(AMDGPU::sub1);
+ MI.eraseFromParent();
+ return BB;
+}
+
+static uint64_t getIdentityValueForWaveReduction(unsigned Opc) {
switch (Opc) {
case AMDGPU::S_MIN_U32:
return std::numeric_limits<uint32_t>::max();
@@ -5583,15 +5644,7 @@ static uint32_t getIdentityValueFor32BitWaveReduction(unsigned Opc) {
return std::numeric_limits<uint32_t>::max();
case AMDGPU::V_MIN_F32_e64:
case AMDGPU::V_MAX_F32_e64:
- return 0x7fc00000; // qNAN
- default:
- llvm_unreachable(
- "Unexpected opcode in getIdentityValueFor32BitWaveReduction");
- }
-}
-
-static uint64_t getIdentityValueFor64BitWaveReduction(unsigned Opc) {
- switch (Opc) {
+ return 0x7fc00000; // qNAN
case AMDGPU::V_CMP_LT_U64_e64: // umin.u64
return std::numeric_limits<uint64_t>::max();
case AMDGPU::V_CMP_LT_I64_e64: // min.i64
@@ -5616,60 +5669,79 @@ static uint64_t getIdentityValueFor64BitWaveReduction(unsigned Opc) {
case AMDGPU::V_ADD_F64_pseudo_e64:
return 0x8000000000000000; // -0.0
default:
- llvm_unreachable(
- "Unexpected opcode in getIdentityValueFor64BitWaveReduction");
- }
-}
-
-static bool is32bitWaveReduceOperation(unsigned Opc) {
- return Opc == AMDGPU::S_MIN_U32 || Opc == AMDGPU::S_MIN_I32 ||
- Opc == AMDGPU::S_MAX_U32 || Opc == AMDGPU::S_MAX_I32 ||
- Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32 ||
- Opc == AMDGPU::S_AND_B32 || Opc == AMDGPU::S_OR_B32 ||
- Opc == AMDGPU::S_XOR_B32 || Opc == AMDGPU::V_MIN_F32_e64 ||
- Opc == AMDGPU::V_MAX_F32_e64 || Opc == AMDGPU::V_ADD_F32_e64 ||
- Opc == AMDGPU::V_SUB_F32_e64;
-}
-
-static bool isFloatingPointWaveReduceOperation(unsigned Opc) {
- return Opc == AMDGPU::V_MIN_F32_e64 || Opc == AMDGPU::V_MAX_F32_e64 ||
- Opc == AMDGPU::V_ADD_F32_e64 || Opc == AMDGPU::V_SUB_F32_e64 ||
- Opc == AMDGPU::V_MIN_F64_e64 || Opc == AMDGPU::V_MAX_F64_e64 ||
- Opc == AMDGPU::V_MIN_NUM_F64_e64 || Opc == AMDGPU::V_MAX_NUM_F64_e64 ||
- Opc == AMDGPU::V_ADD_F64_e64 || Opc == AMDGPU::V_ADD_F64_pseudo_e64;
-}
-
-static unsigned getDPPOpcForWaveReduction(unsigned Opc,
- const GCNSubtarget &ST) {
+ llvm_unreachable("Unexpected opcode in getIdentityValueForWaveReduction");
+ }
+}
+
+static std::tuple<bool, bool> ClassifyWaveReductionOp(unsigned Opc) {
+ bool is32BitOpc = Opc == AMDGPU::S_MIN_U32 || Opc == AMDGPU::S_MIN_I32 ||
+ Opc == AMDGPU::S_MAX_U32 || Opc == AMDGPU::S_MAX_I32 ||
+ Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32 ||
+ Opc == AMDGPU::S_AND_B32 || Opc == AMDGPU::S_OR_B32 ||
+ Opc == AMDGPU::S_XOR_B32 || Opc == AMDGPU::V_MIN_F32_e64 ||
+ Opc == AMDGPU::V_MAX_F32_e64 ||
+ Opc == AMDGPU::V_ADD_F32_e64 ||
+ Opc == AMDGPU::V_SUB_F32_e64;
+ bool isFPOp =
+ Opc == AMDGPU::V_MIN_F32_e64 || Opc == AMDGPU::V_MAX_F32_e64 ||
+ Opc == AMDGPU::V_ADD_F32_e64 || Opc == AMDGPU::V_SUB_F32_e64 ||
+ Opc == AMDGPU::V_MIN_F64_e64 || Opc == AMDGPU::V_MAX_F64_e64 ||
+ Opc == AMDGPU::V_MIN_NUM_F64_e64 || Opc == AMDGPU::V_MAX_NUM_F64_e64 ||
+ Opc == AMDGPU::V_ADD_F64_e64 || Opc == AMDGPU::V_ADD_F64_pseudo_e64;
+ return {is32BitOpc, isFPOp};
+}
+
+static std::tuple<unsigned, unsigned>
+getDPPOpcForWaveReduction(unsigned Opc, const GCNSubtarget &ST) {
+ unsigned DPPOpc;
switch (Opc) {
case AMDGPU::S_MIN_U32:
- return AMDGPU::V_MIN_U32_dpp;
+ DPPOpc = AMDGPU::V_MIN_U32_dpp;
+ break;
case AMDGPU::S_MIN_I32:
- return AMDGPU::V_MIN_I32_dpp;
+ DPPOpc = AMDGPU::V_MIN_I32_dpp;
+ break;
case AMDGPU::S_MAX_U32:
- return AMDGPU::V_MAX_U32_dpp;
+ DPPOpc = AMDGPU::V_MAX_U32_dpp;
+ break;
case AMDGPU::S_MAX_I32:
- return AMDGPU::V_MAX_I32_dpp;
+ DPPOpc = AMDGPU::V_MAX_I32_dpp;
+ break;
case AMDGPU::S_ADD_I32:
case AMDGPU::S_SUB_I32:
- return ST.hasAddNoCarryInsts() ? AMDGPU::V_ADD_U32_dpp
- : AMDGPU::V_ADD_CO_U32_dpp;
+ DPPOpc = ST.hasAddNoCarryInsts() ? AMDGPU::V_ADD_U32_dpp
+ : AMDGPU::V_ADD_CO_U32_dpp;
+ break;
case AMDGPU::S_AND_B32:
- return AMDGPU::V_AND_B32_dpp;
+ DPPOpc = AMDGPU::V_AND_B32_dpp;
+ break;
case AMDGPU::S_OR_B32:
- return AMDGPU::V_OR_B32_dpp;
+ DPPOpc = AMDGPU::V_OR_B32_dpp;
+ break;
case AMDGPU::S_XOR_B32:
- return AMDGPU::V_XOR_B32_dpp;
+ DPPOpc = AMDGPU::V_XOR_B32_dpp;
+ break;
case AMDGPU::V_ADD_F32_e64:
case AMDGPU::V_SUB_F32_e64:
- return AMDGPU::V_ADD_F32_dpp;
+ DPPOpc = AMDGPU::V_ADD_F32_dpp;
+ break;
case AMDGPU::V_MIN_F32_e64:
- return AMDGPU::V_MIN_F32_dpp;
+ DPPOpc = AMDGPU::V_MIN_F32_dpp;
+ break;
case AMDGPU::V_MAX_F32_e64:
- return AMDGPU::V_MAX_F32_dpp;
+ DPPOpc = AMDGPU::V_MAX_F32_dpp;
+ break;
default:
llvm_unreachable("unhandled lane op");
}
+ bool isFPOp = std::get<1>(ClassifyWaveReductionOp(Opc));
+ unsigned ClampOpc = Opc;
+ if (!isFPOp) {
+ if (Opc == AMDGPU::S_SUB_I32)
+ ClampOpc = AMDGPU::S_ADD_I32;
+ ClampOpc = ST.getInstrInfo()->getVALUOp(ClampOpc);
+ }
+ return {DPPOpc, ClampOpc};
}
static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
@@ -5688,6 +5760,28 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
unsigned Stratergy = static_cast<unsigned>(MI.getOperand(2).getImm());
enum WAVE_REDUCE_STRATEGY : unsigned { DEFAULT = 0, ITERATIVE = 1, DPP = 2 };
MachineBasicBlock *RetBB = nullptr;
+ auto ExtractSubRegs = [&](MachineInstr &MI, MachineOperand &Op,
+ const TargetRegisterClass *SrcRC)
+ -> std::tuple<MachineOperand, MachineOperand> {
+ const TargetRegisterClass *SrcSubRC =
+ TRI->getSubRegisterClass(SrcRC, AMDGPU::sub0);
+ MachineOperand Op1L = TII->buildExtractSubRegOrImm(MI, MRI, Op, SrcRC,
+ AMDGPU::sub0, SrcSubRC);
+ MachineOperand Op1H = TII->buildExtractSubRegOrImm(MI, MRI, Op, SrcRC,
+ AMDGPU::sub1, SrcSubRC);
+ return {Op1L, Op1H};
+ };
+ auto BuildRegSequence = [&](MachineBasicBlock &BB,
+ MachineBasicBlock::iterator MI, Register Dst,
+ Register Src0, Register Src1) {
+ auto RegSequence =
+ BuildMI(BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), Dst)
+ .addReg(Src0)
+ .addImm(AMDGPU::sub0)
+ .addReg(Src1)
+ .addImm(AMDGPU::sub1);
+ return RegSequence;
+ };
if (isSGPR) {
switch (Opc) {
case AMDGPU::S_MIN_U32:
@@ -5769,29 +5863,15 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
Register DestSub1 =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
-
- const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg);
- const TargetRegisterClass *SrcSubRC =
- TRI->getSubRegisterClass(SrcRC, AMDGPU::sub0);
-
- MachineOperand Op1L = TII->buildExtractSubRegOrImm(
- MI, MRI, MI.getOperand(1), SrcRC, AMDGPU::sub0, SrcSubRC);
- MachineOperand Op1H = TII->buildExtractSubRegOrImm(
- MI, MRI, MI.getOperand(1), SrcRC, AMDGPU::sub1, SrcSubRC);
-
+ auto [Op1L, Op1H] =
+ ExtractSubRegs(MI, MI.getOperand(1), MRI.getRegClass(SrcReg));
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DestSub0)
.add(Op1L)
.addReg(ParityRegister);
-
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DestSub1)
.add(Op1H)
.addReg(ParityRegister);
-
- BuildMI(BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), DstReg)
- .addReg(DestSub0)
- .addImm(AMDGPU::sub0)
- .addReg(DestSub1)
- .addImm(AMDGPU::sub1);
+ BuildRegSequence(BB, MI, DstReg, DestSub0, DestSub1);
}
break;
}
@@ -5827,16 +5907,8 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
Register NegatedValHi =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
-
- const TargetRegisterClass *Src1RC = MRI.getRegClass(SrcReg);
- const TargetRegisterClass *Src1SubRC =
- TRI->getSubRegisterClass(Src1RC, AMDGPU::sub0);
-
- MachineOperand Op1L = TII->buildExtractSubRegOrImm(
- MI, MRI, MI.getOperand(1), Src1RC, AMDGPU::sub0, Src1SubRC);
- MachineOperand Op1H = TII->buildExtractSubRegOrImm(
- MI, MRI, MI.getOperand(1), Src1RC, AMDGPU::sub1, Src1SubRC);
-
+ auto [Op1L, Op1H] =
+ ExtractSubRegs(MI, MI.getOperand(1), MRI.getRegClass(SrcReg));
if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_SUB_I32), NegatedValLo)
.addImm(0)
@@ -5875,18 +5947,14 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
.addReg(Op1L_Op0H_Reg)
.setOperandDead(3); // Dead scc
}
- BuildMI(BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), DstReg)
- .addReg(DestSub0)
- .addImm(AMDGPU::sub0)
- .addReg(DestSub1)
- .addImm(AMDGPU::sub1);
+ BuildRegSequence(BB, MI, DstReg, DestSub0, DestSub1);
break;
}
case AMDGPU::V_ADD_F32_e64:
case AMDGPU::V_ADD_F64_e64:
case AMDGPU::V_ADD_F64_pseudo_e64:
case AMDGPU::V_SUB_F32_e64: {
- bool is32BitOpc = is32bitWaveReduceOperation(Opc);
+ bool is32BitOpc = std::get<0>(ClassifyWaveReductionOp(Opc));
const TargetRegisterClass *VregRC = TII->getRegClass(TII->get(Opc), 0);
Register ActiveLanesVreg = MRI.createVirtualRegister(VregRC);
Register DstVreg = MRI.createVirtualRegister(VregRC);
@@ -5925,14 +5993,8 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
Register LaneValueHiReg =
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
- const TargetRegisterClass *VregSubRC =
- TRI->getSubRegisterClass(VregRC, AMDGPU::sub0);
- MachineOperand Op1L =
- TII->buildExtractSubRegOrImm(MI, MRI, DestVregInst->getOperand(0),
- VregRC, AMDGPU::sub0, VregSubRC);
- MachineOperand Op1H =
- TII->buildExtractSubRegOrImm(MI, MRI, DestVregInst->getOperand(0),
- VregRC, AMDGPU::sub1, VregSubRC);
+ auto [Op1L, Op1H] =
+ ExtractSubRegs(MI, DestVregInst->getOperand(0), VregRC);
// lane value input should be in an sgpr
BuildMI(BB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
LaneValueLoReg)
@@ -5941,11 +6003,7 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
LaneValueHiReg)
.add(Op1H);
NewAccumulator =
- BuildMI(BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), DstReg)
- .addReg(LaneValueLoReg)
- .addImm(AMDGPU::sub0)
- .addReg(LaneValueHiReg)
- .addImm(AMDGPU::sub1);
+ BuildRegSequence(BB, MI, DstReg, LaneValueLoReg, LaneValueHiReg);
}
}
}
@@ -5955,8 +6013,7 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
} else {
MachineBasicBlock::iterator I = BB.end();
Register SrcReg = MI.getOperand(1).getReg();
- bool is32BitOpc = is32bitWaveReduceOperation(Opc);
- bool isFPOp = isFloatingPointWaveReduceOperation(Opc);
+ auto [is32BitOpc, isFPOp] = ClassifyWaveReductionOp(Opc);
// Create virtual registers required for lowering.
const TargetRegisterClass *WaveMaskRegClass = TRI->getWaveMaskRegClass();
const TargetRegisterClass *DstRegClass = MRI.getRegClass(DstReg);
@@ -5989,19 +6046,15 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
// Create initial values of induction variable from Exec, Accumulator and
// insert branch instr to newly created ComputeBlock
BuildMI(BB, I, DL, TII->get(MovOpcForExec), LoopIterator).addReg(ExecReg);
- if (is32BitOpc) {
- uint32_t IdentityValue = getIdentityValueFor32BitWaveReduction(Opc);
- BuildMI(BB, I, DL, TII->get(AMDGPU::S_MOV_B32), IdentityValReg)
- .addImm(IdentityValue);
- } else {
- uint64_t IdentityValue =
- MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64
- ? 0x0 // +0.0 for double sub reduction
- : getIdentityValueFor64BitWaveReduction(Opc);
- BuildMI(BB, I, DL, TII->get(AMDGPU::S_MOV_B64_IMM_PSEUDO),
- IdentityValReg)
- .addImm(IdentityValue);
- }
+ uint64_t IdentityValue =
+ MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64
+ ? 0x0 // +0.0 for double sub reduction
+ : getIdentityValueForWaveReduction(Opc);
+ BuildMI(BB, I, DL,
+ TII->get(is32BitOpc ? AMDGPU::S_MOV_B32
+ : AMDGPU::S_MOV_B64_IMM_PSEUDO),
+ IdentityValReg)
+ .addImm(IdentityValue);
// clang-format off
BuildMI(BB, I, DL, TII->get(AMDGPU::S_BRANCH))
.addMBB(ComputeLoop);
@@ -6061,13 +6114,8 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
Register LaneValReg =
MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
- const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg);
- const TargetRegisterClass *SrcSubRC =
- TRI->getSubRegisterClass(SrcRC, AMDGPU::sub0);
- MachineOperand Op1L = TII->buildExtractSubRegOrImm(
- MI, MRI, MI.getOperand(1), SrcRC, AMDGPU::sub0, SrcSubRC);
- MachineOperand Op1H = TII->buildExtractSubRegOrImm(
- MI, MRI, MI.getOperand(1), SrcRC, AMDGPU::sub1, SrcSubRC);
+ auto [Op1L, Op1H] =
+ ExtractSubRegs(MI, MI.getOperand(1), MRI.getRegClass(SrcReg));
// lane value input should be in an sgpr
BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::V_READLANE_B32),
LaneValueLoReg)
@@ -6077,13 +6125,8 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
LaneValueHiReg)
.add(Op1H)
.addReg(FF1Reg);
- auto LaneValue =
- BuildMI(*ComputeLoop, I, DL, TII->get(TargetOpcode::REG_SEQUENCE),
- LaneValReg)
- .addReg(LaneValueLoReg)
- .addImm(AMDGPU::sub0)
- .addReg(LaneValueHiReg)
- .addImm(AMDGPU::sub1);
+ auto LaneValue = BuildRegSequence(*ComputeLoop, I, LaneValReg,
+ LaneValueLoReg, LaneValueHiReg);
switch (Opc) {
case AMDGPU::S_OR_B64:
case AMDGPU::S_AND_B64:
@@ -6105,21 +6148,11 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src);
const TargetRegisterClass *VregClass =
TRI->getAllocatableClass(TII->getRegClass(MI.getDesc(), SrcIdx));
- const TargetRegisterClass *VSubRegClass =
- TRI->getSubRegisterClass(VregClass, AMDGPU::sub0);
Register AccumulatorVReg = MRI.createVirtualRegister(VregClass);
- MachineOperand SrcReg0Sub0 = TII->buildExtractSubRegOrImm(
- MI, MRI, Accumulator->getOperand(0), VregClass, AMDGPU::sub0,
- VSubRegClass);
- MachineOperand SrcReg0Sub1 = TII->buildExtractSubRegOrImm(
- MI, MRI, Accumulator->getOperand(0), VregClass, AMDGPU::sub1,
- VSubRegClass);
- BuildMI(*ComputeLoop, I, DL, TII->get(TargetOpcode::REG_SEQUENCE),
- AccumulatorVReg)
- .add(SrcReg0Sub0)
- .addImm(AMDGPU::sub0)
- .add(SrcReg0Sub1)
- .addImm(AMDGPU::sub1);
+ auto [SrcReg0Sub0, SrcReg0Sub1] =
+ ExtractSubRegs(MI, Accumulator->getOperand(0), VregClass);
+ BuildRegSequence(*ComputeLoop, I, AccumulatorVReg,
+ SrcReg0Sub0.getReg(), SrcReg0Sub1.getReg());
BuildMI(*ComputeLoop, I, DL, TII->get(Opc), LaneMaskReg)
.addReg(LaneValue->getOperand(0).getReg())
.addReg(AccumulatorVReg);
@@ -6145,8 +6178,6 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src);
const TargetRegisterClass *VregRC =
TRI->getAllocatableClass(TII->getRegClass(MI.getDesc(), SrcIdx));
- const TargetRegisterClass *VregSubRC =
- TRI->getSubRegisterClass(VregRC, AMDGPU::sub0);
Register AccumulatorVReg = MRI.createVirtualRegister(VregRC);
Register DstVreg = MRI.createVirtualRegister(VregRC);
Register LaneValLo =
@@ -6174,20 +6205,12 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
BuildMI(*ComputeLoop, I, DL,
TII->get(AMDGPU::V_READFIRSTLANE_B32), LaneValHi);
MachineBasicBlock::iterator Iters = *ReadLaneLo;
- MachineOperand Op1L = TII->buildExtractSubRegOrImm(
- Iters, MRI, DstVregInst->getOperand(0), VregRC, AMDGPU::sub0,
- VregSubRC);
- MachineOperand Op1H = TII->buildExtractSubRegOrImm(
- Iters, MRI, DstVregInst->getOperand(0), VregRC, AMDGPU::sub1,
- VregSubRC);
+ auto [Op1L, Op1H] =
+ ExtractSubRegs(*Iters, DstVregInst->getOperand(0), VregRC);
ReadLaneLo.add(Op1L);
ReadLaneHi.add(Op1H);
- NewAccumulator = BuildMI(*ComputeLoop, I, DL,
- TII->get(TargetOpcode::REG_SEQUENCE), DstReg)
- .addReg(LaneValLo)
- .addImm(AMDGPU::sub0)
- .addReg(LaneValHi)
- .addImm(AMDGPU::sub1);
+ NewAccumulator =
+ BuildRegSequence(*ComputeLoop, I, DstReg, LaneValLo, LaneValHi);
break;
}
case AMDGPU::S_ADD_U64_PSEUDO:
@@ -6224,7 +6247,6 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
} else {
assert(ST.hasDPP() && "Sub Target does not support DPP Operations");
- bool IsFPOp = isFloatingPointWaveReduceOperation(Opc);
Register SrcWithIdentity = MRI.createVirtualRegister(SrcRegClass);
Register IdentityVGPR = MRI.createVirtualRegister(SrcRegClass);
Register IdentitySGPR = MRI.createVirtualRegister(DstRegClass);
@@ -6240,29 +6262,30 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
Register FinalDPPResult;
BuildMI(BB, MI, DL, TII->get(AMDGPU::IMPLICIT_DEF), UndefExec);
- uint32_t IdentityValue = getIdentityValueFor32BitWaveReduction(Opc);
+ uint64_t IdentityValue = getIdentityValueForWaveReduction(Opc);
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B32), IdentitySGPR)
.addImm(IdentityValue);
BuildMI(BB, MI, DL, TII->get(AMDGPU::COPY), IdentityVGPR)
.addReg(IdentitySGPR);
-
- // Set inactive lanes to the identity value.
- BuildMI(BB, MI, DL, TII->get(AMDGPU::V_SET_INACTIVE_B32), SrcWithIdentity)
- .addImm(0) // src0 modifiers
- .addReg(SrcReg) // src0
- .addImm(0) // src1 modifiers
- .addReg(IdentityVGPR) // identity value for inactive lanes
- .addReg(UndefExec); // bool i1
-
- unsigned DPPOpc = getDPPOpcForWaveReduction(Opc, ST);
+ auto [DPPOpc, ClampOpc] = getDPPOpcForWaveReduction(Opc, ST);
+ auto BuildSetInactiveInstr = [&](Register Dst, Register Src0,
+ Register Src1) {
+ return BuildMI(BB, MI, DL, TII->get(AMDGPU::V_SET_INACTIVE_B32),
+ Dst)
+ .addImm(0) // src0 modifiers
+ .addReg(Src0) // src0
+ .addImm(0) // src1 modifiers
+ .addReg(Src1) // identity value for inactive lanes
+ .addReg(UndefExec); // bool i1
+ };
auto BuildDPPMachineInstr = [&](Register Dst, Register Src,
unsigned DPPCtrl) {
auto DPPInstr =
BuildMI(BB, MI, DL, TII->get(DPPOpc), Dst).addReg(Src); // old
- if (IsFPOp)
+ if (isFPOp)
DPPInstr.addImm(SISrcMods::NONE); // src0 modifier
DPPInstr.addReg(Src); // src0
- if (IsFPOp)
+ if (isFPOp)
DPPInstr.addImm(SISrcMods::NONE); // src1 modifier
DPPInstr
.addReg(Src) // src1
@@ -6272,24 +6295,23 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
.addImm(0); // bound-control
};
auto BuildClampInstr = [&](Register Dst, Register Src0, Register Src1) {
- unsigned ClampOpc = Opc;
- if (!IsFPOp) {
- if (Opc == AMDGPU::S_SUB_I32)
- ClampOpc = AMDGPU::S_ADD_I32;
- ClampOpc = TII->getVALUOp(ClampOpc);
- }
auto ClampInstr = BuildMI(BB, MI, DL, TII->get(ClampOpc), Dst);
- if (IsFPOp)
+ if (isFPOp)
ClampInstr.addImm(SISrcMods::NONE); // src0 mod
ClampInstr.addReg(Src0); // src0
- if (IsFPOp)
+ if (isFPOp)
ClampInstr.addImm(SISrcMods::NONE); // src1 mod
ClampInstr.addReg(Src1); // src1
if (TII->hasIntClamp(*ClampInstr) || TII->hasFPClamp(*ClampInstr))
ClampInstr.addImm(0); // clamp
- if (IsFPOp)
+ if (isFPOp)
ClampInstr.addImm(0); // omod
};
+
+ // Set inactive lanes to the identity value.
+ MachineInstr *SrcWithIdentityInstr =
+ BuildSetInactiveInstr(SrcWithIdentity, SrcReg, IdentityVGPR);
+
// DPP reduction
BuildDPPMachineInstr(DPPRowShr1, SrcWithIdentity,
AMDGPU::DPP::ROW_SHR_FIRST);
@@ -6767,59 +6789,7 @@ SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
case AMDGPU::SI_KILL_I1_PSEUDO:
return splitKillBlock(MI, BB);
case AMDGPU::V_CNDMASK_B64_PSEUDO: {
- Register Dst = MI.getOperand(0).getReg();
- const MachineOperand &Src0 = MI.getOperand(1);
- const MachineOperand &Src1 = MI.getOperand(2);
- Register SrcCond = MI.getOperand(3).getReg();
-
- Register DstLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
- Register DstHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
- const auto *CondRC = TRI->getWaveMaskRegClass();
- Register SrcCondCopy = MRI.createVirtualRegister(CondRC);
-
- const TargetRegisterClass *Src0RC = Src0.isReg()
- ? MRI.getRegClass(Src0.getReg())
- : &AMDGPU::VReg_64RegClass;
- const TargetRegisterClass *Src1RC = Src1.isReg()
- ? MRI.getRegClass(Src1.getReg())
- : &AMDGPU::VReg_64RegClass;
-
- const TargetRegisterClass *Src0SubRC =
- TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
- const TargetRegisterClass *Src1SubRC =
- TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
-
- MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(
- MI, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC);
- MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(
- MI, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC);
-
- MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(
- MI, MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC);
- MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
- MI, MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC);
-
- BuildMI(*BB, MI, DL, TII->get(AMDGPU::COPY), SrcCondCopy).addReg(SrcCond);
- BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstLo)
- .addImm(0)
- .add(Src0Sub0)
- .addImm(0)
- .add(Src1Sub0)
- .addReg(SrcCondCopy);
- BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstHi)
- .addImm(0)
- .add(Src0Sub1)
- .addImm(0)
- .add(Src1Sub1)
- .addReg(SrcCondCopy);
-
- BuildMI(*BB, MI, DL, TII->get(AMDGPU::REG_SEQUENCE), Dst)
- .addReg(DstLo)
- .addImm(AMDGPU::sub0)
- .addReg(DstHi)
- .addImm(AMDGPU::sub1);
- MI.eraseFromParent();
- return BB;
+ return Expand64BitV_CND_MASK(MI, BB);
}
case AMDGPU::SI_BR_UNDEF: {
MachineInstr *Br = BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_CBRANCH_SCC1))
>From 4259dc2c6b9550ed51c3f2027b618e68b1cc30af Mon Sep 17 00:00:00 2001
From: Aaditya <Aaditya.AlokDeshpande at amd.com>
Date: Tue, 7 Apr 2026 14:24:41 +0530
Subject: [PATCH 2/6] Use MachineInstrBuilder instead of BuildMI. Return
registers by value rather than MachineOperands. Minor Code Cleanup/
Refactoring.
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 117 ++++++++++++----------
1 file changed, 63 insertions(+), 54 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 465bf3981cd5a..611154fa62ad8 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -32,6 +32,7 @@
#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
+#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
@@ -5509,7 +5510,7 @@ static MachineBasicBlock *emitIndirectDst(MachineInstr &MI,
return LoopBB;
}
-static MachineBasicBlock *Expand64BitScalarArithmetic(MachineInstr &MI,
+static MachineBasicBlock *expand64BitScalarArithmetic(MachineInstr &MI,
MachineBasicBlock *BB) {
// For targets older than GFX12, we emit a sequence of 32-bit operations.
// For GFX12, we emit s_add_u64 and s_sub_u64.
@@ -5560,14 +5561,12 @@ static MachineBasicBlock *Expand64BitScalarArithmetic(MachineInstr &MI,
return BB;
}
-static MachineBasicBlock *Expand64BitV_CND_MASK(MachineInstr &MI,
- MachineBasicBlock *BB) {
+static void expand64BitV_CNDMASK(MachineInstr &MI, MachineBasicBlock *BB) {
MachineFunction *MF = BB->getParent();
const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
const SIInstrInfo *TII = ST.getInstrInfo();
const SIRegisterInfo *TRI = ST.getRegisterInfo();
MachineRegisterInfo &MRI = MF->getRegInfo();
- const DebugLoc &DL = MI.getDebugLoc();
Register Dst = MI.getOperand(0).getReg();
const MachineOperand &Src0 = MI.getOperand(1);
const MachineOperand &Src1 = MI.getOperand(2);
@@ -5578,10 +5577,14 @@ static MachineBasicBlock *Expand64BitV_CND_MASK(MachineInstr &MI,
const auto *CondRC = TRI->getWaveMaskRegClass();
Register SrcCondCopy = MRI.createVirtualRegister(CondRC);
+ int Src0Idx =
+ AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0);
+ int Src1Idx =
+ AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src1);
const TargetRegisterClass *Src0RC =
- Src0.isReg() ? MRI.getRegClass(Src0.getReg()) : &AMDGPU::VReg_64RegClass;
+ TRI->getAllocatableClass(TII->getRegClass(MI.getDesc(), Src0Idx));
const TargetRegisterClass *Src1RC =
- Src1.isReg() ? MRI.getRegClass(Src1.getReg()) : &AMDGPU::VReg_64RegClass;
+ TRI->getAllocatableClass(TII->getRegClass(MI.getDesc(), Src1Idx));
const TargetRegisterClass *Src0SubRC =
TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
@@ -5598,27 +5601,31 @@ static MachineBasicBlock *Expand64BitV_CND_MASK(MachineInstr &MI,
MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
MI, MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC);
- BuildMI(*BB, MI, DL, TII->get(AMDGPU::COPY), SrcCondCopy).addReg(SrcCond);
- BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstLo)
+ MachineIRBuilder B(MI);
+ B.buildInstr(AMDGPU::COPY).addDef(SrcCondCopy).addReg(SrcCond);
+ B.buildInstr(AMDGPU::V_CNDMASK_B32_e64)
+ .addDef(DstLo)
.addImm(0)
.add(Src0Sub0)
.addImm(0)
.add(Src1Sub0)
.addReg(SrcCondCopy);
- BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstHi)
+
+ B.buildInstr(AMDGPU::V_CNDMASK_B32_e64)
+ .addDef(DstHi)
.addImm(0)
.add(Src0Sub1)
.addImm(0)
.add(Src1Sub1)
.addReg(SrcCondCopy);
- BuildMI(*BB, MI, DL, TII->get(AMDGPU::REG_SEQUENCE), Dst)
+ B.buildInstr(TargetOpcode::REG_SEQUENCE)
+ .addDef(Dst)
.addReg(DstLo)
.addImm(AMDGPU::sub0)
.addReg(DstHi)
.addImm(AMDGPU::sub1);
MI.eraseFromParent();
- return BB;
}
static uint64_t getIdentityValueForWaveReduction(unsigned Opc) {
@@ -5673,22 +5680,22 @@ static uint64_t getIdentityValueForWaveReduction(unsigned Opc) {
}
}
-static std::tuple<bool, bool> ClassifyWaveReductionOp(unsigned Opc) {
- bool is32BitOpc = Opc == AMDGPU::S_MIN_U32 || Opc == AMDGPU::S_MIN_I32 ||
- Opc == AMDGPU::S_MAX_U32 || Opc == AMDGPU::S_MAX_I32 ||
- Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32 ||
- Opc == AMDGPU::S_AND_B32 || Opc == AMDGPU::S_OR_B32 ||
- Opc == AMDGPU::S_XOR_B32 || Opc == AMDGPU::V_MIN_F32_e64 ||
- Opc == AMDGPU::V_MAX_F32_e64 ||
- Opc == AMDGPU::V_ADD_F32_e64 ||
- Opc == AMDGPU::V_SUB_F32_e64;
- bool isFPOp =
- Opc == AMDGPU::V_MIN_F32_e64 || Opc == AMDGPU::V_MAX_F32_e64 ||
- Opc == AMDGPU::V_ADD_F32_e64 || Opc == AMDGPU::V_SUB_F32_e64 ||
- Opc == AMDGPU::V_MIN_F64_e64 || Opc == AMDGPU::V_MAX_F64_e64 ||
- Opc == AMDGPU::V_MIN_NUM_F64_e64 || Opc == AMDGPU::V_MAX_NUM_F64_e64 ||
- Opc == AMDGPU::V_ADD_F64_e64 || Opc == AMDGPU::V_ADD_F64_pseudo_e64;
- return {is32BitOpc, isFPOp};
+static bool is32bitWaveReduceOperation(unsigned Opc) {
+ return Opc == AMDGPU::S_MIN_U32 || Opc == AMDGPU::S_MIN_I32 ||
+ Opc == AMDGPU::S_MAX_U32 || Opc == AMDGPU::S_MAX_I32 ||
+ Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32 ||
+ Opc == AMDGPU::S_AND_B32 || Opc == AMDGPU::S_OR_B32 ||
+ Opc == AMDGPU::S_XOR_B32 || Opc == AMDGPU::V_MIN_F32_e64 ||
+ Opc == AMDGPU::V_MAX_F32_e64 || Opc == AMDGPU::V_ADD_F32_e64 ||
+ Opc == AMDGPU::V_SUB_F32_e64;
+}
+
+static bool isFloatingPointWaveReduceOperation(unsigned Opc) {
+ return Opc == AMDGPU::V_MIN_F32_e64 || Opc == AMDGPU::V_MAX_F32_e64 ||
+ Opc == AMDGPU::V_ADD_F32_e64 || Opc == AMDGPU::V_SUB_F32_e64 ||
+ Opc == AMDGPU::V_MIN_F64_e64 || Opc == AMDGPU::V_MAX_F64_e64 ||
+ Opc == AMDGPU::V_MIN_NUM_F64_e64 || Opc == AMDGPU::V_MAX_NUM_F64_e64 ||
+ Opc == AMDGPU::V_ADD_F64_e64 || Opc == AMDGPU::V_ADD_F64_pseudo_e64;
}
static std::tuple<unsigned, unsigned>
@@ -5734,7 +5741,7 @@ getDPPOpcForWaveReduction(unsigned Opc, const GCNSubtarget &ST) {
default:
llvm_unreachable("unhandled lane op");
}
- bool isFPOp = std::get<1>(ClassifyWaveReductionOp(Opc));
+ bool isFPOp = isFloatingPointWaveReduceOperation(Opc);
unsigned ClampOpc = Opc;
if (!isFPOp) {
if (Opc == AMDGPU::S_SUB_I32)
@@ -5760,15 +5767,15 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
unsigned Stratergy = static_cast<unsigned>(MI.getOperand(2).getImm());
enum WAVE_REDUCE_STRATEGY : unsigned { DEFAULT = 0, ITERATIVE = 1, DPP = 2 };
MachineBasicBlock *RetBB = nullptr;
- auto ExtractSubRegs = [&](MachineInstr &MI, MachineOperand &Op,
- const TargetRegisterClass *SrcRC)
- -> std::tuple<MachineOperand, MachineOperand> {
+ auto ExtractSubRegs =
+ [&](MachineInstr &MI, MachineOperand &Op,
+ const TargetRegisterClass *SrcRC) -> std::pair<Register, Register> {
const TargetRegisterClass *SrcSubRC =
TRI->getSubRegisterClass(SrcRC, AMDGPU::sub0);
- MachineOperand Op1L = TII->buildExtractSubRegOrImm(MI, MRI, Op, SrcRC,
- AMDGPU::sub0, SrcSubRC);
- MachineOperand Op1H = TII->buildExtractSubRegOrImm(MI, MRI, Op, SrcRC,
- AMDGPU::sub1, SrcSubRC);
+ Register Op1L =
+ TII->buildExtractSubReg(MI, MRI, Op, SrcRC, AMDGPU::sub0, SrcSubRC);
+ Register Op1H =
+ TII->buildExtractSubReg(MI, MRI, Op, SrcRC, AMDGPU::sub1, SrcSubRC);
return {Op1L, Op1H};
};
auto BuildRegSequence = [&](MachineBasicBlock &BB,
@@ -5866,10 +5873,10 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
auto [Op1L, Op1H] =
ExtractSubRegs(MI, MI.getOperand(1), MRI.getRegClass(SrcReg));
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DestSub0)
- .add(Op1L)
+ .addReg(Op1L)
.addReg(ParityRegister);
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DestSub1)
- .add(Op1H)
+ .addReg(Op1H)
.addReg(ParityRegister);
BuildRegSequence(BB, MI, DstReg, DestSub0, DestSub1);
}
@@ -5919,20 +5926,20 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
.addImm(31)
.setOperandDead(3); // Dead scc
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), Op1L_Op0H_Reg)
- .add(Op1L)
+ .addReg(Op1L)
.addReg(NegatedValHi);
}
Register LowOpcode = Opc == AMDGPU::S_SUB_U64_PSEUDO
? NegatedValLo
: NewAccumulator->getOperand(0).getReg();
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DestSub0)
- .add(Op1L)
+ .addReg(Op1L)
.addReg(LowOpcode);
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_HI_U32), CarryReg)
- .add(Op1L)
+ .addReg(Op1L)
.addReg(LowOpcode);
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), Op1H_Op0L_Reg)
- .add(Op1H)
+ .addReg(Op1H)
.addReg(LowOpcode);
Register HiVal = Opc == AMDGPU::S_SUB_U64_PSEUDO ? AddReg : DestSub1;
@@ -5954,7 +5961,7 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
case AMDGPU::V_ADD_F64_e64:
case AMDGPU::V_ADD_F64_pseudo_e64:
case AMDGPU::V_SUB_F32_e64: {
- bool is32BitOpc = std::get<0>(ClassifyWaveReductionOp(Opc));
+ bool is32BitOpc = is32bitWaveReduceOperation(Opc);
const TargetRegisterClass *VregRC = TII->getRegClass(TII->get(Opc), 0);
Register ActiveLanesVreg = MRI.createVirtualRegister(VregRC);
Register DstVreg = MRI.createVirtualRegister(VregRC);
@@ -5998,10 +6005,10 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
// lane value input should be in an sgpr
BuildMI(BB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
LaneValueLoReg)
- .add(Op1L);
+ .addReg(Op1L);
BuildMI(BB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
LaneValueHiReg)
- .add(Op1H);
+ .addReg(Op1H);
NewAccumulator =
BuildRegSequence(BB, MI, DstReg, LaneValueLoReg, LaneValueHiReg);
}
@@ -6013,7 +6020,8 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
} else {
MachineBasicBlock::iterator I = BB.end();
Register SrcReg = MI.getOperand(1).getReg();
- auto [is32BitOpc, isFPOp] = ClassifyWaveReductionOp(Opc);
+ bool is32BitOpc = is32bitWaveReduceOperation(Opc);
+ bool isFPOp = isFloatingPointWaveReduceOperation(Opc);
// Create virtual registers required for lowering.
const TargetRegisterClass *WaveMaskRegClass = TRI->getWaveMaskRegClass();
const TargetRegisterClass *DstRegClass = MRI.getRegClass(DstReg);
@@ -6119,11 +6127,11 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
// lane value input should be in an sgpr
BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::V_READLANE_B32),
LaneValueLoReg)
- .add(Op1L)
+ .addReg(Op1L)
.addReg(FF1Reg);
BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::V_READLANE_B32),
LaneValueHiReg)
- .add(Op1H)
+ .addReg(Op1H)
.addReg(FF1Reg);
auto LaneValue = BuildRegSequence(*ComputeLoop, I, LaneValReg,
LaneValueLoReg, LaneValueHiReg);
@@ -6151,8 +6159,8 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
Register AccumulatorVReg = MRI.createVirtualRegister(VregClass);
auto [SrcReg0Sub0, SrcReg0Sub1] =
ExtractSubRegs(MI, Accumulator->getOperand(0), VregClass);
- BuildRegSequence(*ComputeLoop, I, AccumulatorVReg,
- SrcReg0Sub0.getReg(), SrcReg0Sub1.getReg());
+ BuildRegSequence(*ComputeLoop, I, AccumulatorVReg, SrcReg0Sub0,
+ SrcReg0Sub1);
BuildMI(*ComputeLoop, I, DL, TII->get(Opc), LaneMaskReg)
.addReg(LaneValue->getOperand(0).getReg())
.addReg(AccumulatorVReg);
@@ -6207,8 +6215,8 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
MachineBasicBlock::iterator Iters = *ReadLaneLo;
auto [Op1L, Op1H] =
ExtractSubRegs(*Iters, DstVregInst->getOperand(0), VregRC);
- ReadLaneLo.add(Op1L);
- ReadLaneHi.add(Op1H);
+ ReadLaneLo.addReg(Op1L);
+ ReadLaneHi.addReg(Op1H);
NewAccumulator =
BuildRegSequence(*ComputeLoop, I, DstReg, LaneValLo, LaneValHi);
break;
@@ -6219,7 +6227,7 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
.addReg(Accumulator->getOperand(0).getReg())
.addReg(LaneValue->getOperand(0).getReg());
ComputeLoop =
- Expand64BitScalarArithmetic(*NewAccumulator, ComputeLoop);
+ expand64BitScalarArithmetic(*NewAccumulator, ComputeLoop);
break;
}
}
@@ -6525,7 +6533,7 @@ SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
}
case AMDGPU::S_ADD_U64_PSEUDO:
case AMDGPU::S_SUB_U64_PSEUDO: {
- return Expand64BitScalarArithmetic(MI, BB);
+ return expand64BitScalarArithmetic(MI, BB);
}
case AMDGPU::V_ADD_U64_PSEUDO:
case AMDGPU::V_SUB_U64_PSEUDO: {
@@ -6789,7 +6797,8 @@ SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
case AMDGPU::SI_KILL_I1_PSEUDO:
return splitKillBlock(MI, BB);
case AMDGPU::V_CNDMASK_B64_PSEUDO: {
- return Expand64BitV_CND_MASK(MI, BB);
+ expand64BitV_CNDMASK(MI, BB);
+ return BB;
}
case AMDGPU::SI_BR_UNDEF: {
MachineInstr *Br = BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_CBRANCH_SCC1))
>From e129d09584ed0ece43674db9fe33737c15009d56 Mon Sep 17 00:00:00 2001
From: Aaditya <Aaditya.AlokDeshpande at amd.com>
Date: Tue, 7 Apr 2026 15:36:28 +0530
Subject: [PATCH 3/6] Avoid capturing the structed binding.
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 611154fa62ad8..4d85a9af6169e 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -6275,7 +6275,9 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
.addImm(IdentityValue);
BuildMI(BB, MI, DL, TII->get(AMDGPU::COPY), IdentityVGPR)
.addReg(IdentitySGPR);
- auto [DPPOpc, ClampOpc] = getDPPOpcForWaveReduction(Opc, ST);
+ auto DPPClampOpcPair = getDPPOpcForWaveReduction(Opc, ST);
+ unsigned DPPOpc = std::get<0>(DPPClampOpcPair);
+ unsigned ClampOpc = std::get<1>(DPPClampOpcPair);
auto BuildSetInactiveInstr = [&](Register Dst, Register Src0,
Register Src1) {
return BuildMI(BB, MI, DL, TII->get(AMDGPU::V_SET_INACTIVE_B32),
>From 0c1b83638d4e99b9a27f3ff923f462f9f4b042b4 Mon Sep 17 00:00:00 2001
From: Aaditya <Aaditya.AlokDeshpande at amd.com>
Date: Wed, 8 Apr 2026 11:04:41 +0530
Subject: [PATCH 4/6] Remove unused variable
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 4d85a9af6169e..caa8f10abb9fe 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -6319,8 +6319,7 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
};
// Set inactive lanes to the identity value.
- MachineInstr *SrcWithIdentityInstr =
- BuildSetInactiveInstr(SrcWithIdentity, SrcReg, IdentityVGPR);
+ BuildSetInactiveInstr(SrcWithIdentity, SrcReg, IdentityVGPR);
// DPP reduction
BuildDPPMachineInstr(DPPRowShr1, SrcWithIdentity,
>From e921c2669f7b8637e3a7269fbac920700362b5d4 Mon Sep 17 00:00:00 2001
From: Aaditya <Aaditya.AlokDeshpande at amd.com>
Date: Wed, 8 Apr 2026 16:42:34 +0530
Subject: [PATCH 5/6] Remove `MachineIRBuilder` uses
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 14 +++++---------
1 file changed, 5 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index caa8f10abb9fe..0f1dbc9584d06 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -32,7 +32,6 @@
#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
-#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
@@ -5567,6 +5566,7 @@ static void expand64BitV_CNDMASK(MachineInstr &MI, MachineBasicBlock *BB) {
const SIInstrInfo *TII = ST.getInstrInfo();
const SIRegisterInfo *TRI = ST.getRegisterInfo();
MachineRegisterInfo &MRI = MF->getRegInfo();
+ const DebugLoc &DL = MI.getDebugLoc();
Register Dst = MI.getOperand(0).getReg();
const MachineOperand &Src0 = MI.getOperand(1);
const MachineOperand &Src1 = MI.getOperand(2);
@@ -5601,26 +5601,22 @@ static void expand64BitV_CNDMASK(MachineInstr &MI, MachineBasicBlock *BB) {
MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
MI, MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC);
- MachineIRBuilder B(MI);
- B.buildInstr(AMDGPU::COPY).addDef(SrcCondCopy).addReg(SrcCond);
- B.buildInstr(AMDGPU::V_CNDMASK_B32_e64)
- .addDef(DstLo)
+ BuildMI(*BB, MI, DL, TII->get(AMDGPU::COPY), SrcCondCopy).addReg(SrcCond);
+ BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstLo)
.addImm(0)
.add(Src0Sub0)
.addImm(0)
.add(Src1Sub0)
.addReg(SrcCondCopy);
- B.buildInstr(AMDGPU::V_CNDMASK_B32_e64)
- .addDef(DstHi)
+ BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstHi)
.addImm(0)
.add(Src0Sub1)
.addImm(0)
.add(Src1Sub1)
.addReg(SrcCondCopy);
- B.buildInstr(TargetOpcode::REG_SEQUENCE)
- .addDef(Dst)
+ BuildMI(*BB, MI, DL, TII->get(AMDGPU::REG_SEQUENCE), Dst)
.addReg(DstLo)
.addImm(AMDGPU::sub0)
.addReg(DstHi)
>From bc17722c669c5ebb37c3f6a57ea229daaf870b9a Mon Sep 17 00:00:00 2001
From: Aaditya <Aaditya.AlokDeshpande at amd.com>
Date: Wed, 8 Apr 2026 17:09:13 +0530
Subject: [PATCH 6/6] Refactor lambda to a helper function
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 50 ++++++++++++-----------
1 file changed, 27 insertions(+), 23 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 0f1dbc9584d06..43cfccfeca652 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -5574,7 +5574,7 @@ static void expand64BitV_CNDMASK(MachineInstr &MI, MachineBasicBlock *BB) {
Register DstLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
Register DstHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
- const auto *CondRC = TRI->getWaveMaskRegClass();
+ const TargetRegisterClass *CondRC = TRI->getWaveMaskRegClass();
Register SrcCondCopy = MRI.createVirtualRegister(CondRC);
int Src0Idx =
@@ -5747,6 +5747,21 @@ getDPPOpcForWaveReduction(unsigned Opc, const GCNSubtarget &ST) {
return {DPPOpc, ClampOpc};
}
+static std::pair<Register, Register>
+ExtractSubRegs(MachineInstr &MI, MachineOperand &Op,
+ const TargetRegisterClass *SrcRC, const GCNSubtarget &ST,
+ MachineRegisterInfo &MRI) {
+ const SIRegisterInfo *TRI = ST.getRegisterInfo();
+ const SIInstrInfo *TII = ST.getInstrInfo();
+ const TargetRegisterClass *SrcSubRC =
+ TRI->getSubRegisterClass(SrcRC, AMDGPU::sub0);
+ Register Op1L =
+ TII->buildExtractSubReg(MI, MRI, Op, SrcRC, AMDGPU::sub0, SrcSubRC);
+ Register Op1H =
+ TII->buildExtractSubReg(MI, MRI, Op, SrcRC, AMDGPU::sub1, SrcSubRC);
+ return {Op1L, Op1H};
+}
+
static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
MachineBasicBlock &BB,
const GCNSubtarget &ST,
@@ -5763,17 +5778,6 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
unsigned Stratergy = static_cast<unsigned>(MI.getOperand(2).getImm());
enum WAVE_REDUCE_STRATEGY : unsigned { DEFAULT = 0, ITERATIVE = 1, DPP = 2 };
MachineBasicBlock *RetBB = nullptr;
- auto ExtractSubRegs =
- [&](MachineInstr &MI, MachineOperand &Op,
- const TargetRegisterClass *SrcRC) -> std::pair<Register, Register> {
- const TargetRegisterClass *SrcSubRC =
- TRI->getSubRegisterClass(SrcRC, AMDGPU::sub0);
- Register Op1L =
- TII->buildExtractSubReg(MI, MRI, Op, SrcRC, AMDGPU::sub0, SrcSubRC);
- Register Op1H =
- TII->buildExtractSubReg(MI, MRI, Op, SrcRC, AMDGPU::sub1, SrcSubRC);
- return {Op1L, Op1H};
- };
auto BuildRegSequence = [&](MachineBasicBlock &BB,
MachineBasicBlock::iterator MI, Register Dst,
Register Src0, Register Src1) {
@@ -5866,8 +5870,8 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
Register DestSub1 =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
- auto [Op1L, Op1H] =
- ExtractSubRegs(MI, MI.getOperand(1), MRI.getRegClass(SrcReg));
+ auto [Op1L, Op1H] = ExtractSubRegs(MI, MI.getOperand(1),
+ MRI.getRegClass(SrcReg), ST, MRI);
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DestSub0)
.addReg(Op1L)
.addReg(ParityRegister);
@@ -5910,8 +5914,8 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
Register NegatedValHi =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
- auto [Op1L, Op1H] =
- ExtractSubRegs(MI, MI.getOperand(1), MRI.getRegClass(SrcReg));
+ auto [Op1L, Op1H] = ExtractSubRegs(MI, MI.getOperand(1),
+ MRI.getRegClass(SrcReg), ST, MRI);
if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_SUB_I32), NegatedValLo)
.addImm(0)
@@ -5997,7 +6001,7 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
Register LaneValueHiReg =
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
auto [Op1L, Op1H] =
- ExtractSubRegs(MI, DestVregInst->getOperand(0), VregRC);
+ ExtractSubRegs(MI, DestVregInst->getOperand(0), VregRC, ST, MRI);
// lane value input should be in an sgpr
BuildMI(BB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
LaneValueLoReg)
@@ -6118,8 +6122,8 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
Register LaneValReg =
MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
- auto [Op1L, Op1H] =
- ExtractSubRegs(MI, MI.getOperand(1), MRI.getRegClass(SrcReg));
+ auto [Op1L, Op1H] = ExtractSubRegs(MI, MI.getOperand(1),
+ MRI.getRegClass(SrcReg), ST, MRI);
// lane value input should be in an sgpr
BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::V_READLANE_B32),
LaneValueLoReg)
@@ -6153,8 +6157,8 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
const TargetRegisterClass *VregClass =
TRI->getAllocatableClass(TII->getRegClass(MI.getDesc(), SrcIdx));
Register AccumulatorVReg = MRI.createVirtualRegister(VregClass);
- auto [SrcReg0Sub0, SrcReg0Sub1] =
- ExtractSubRegs(MI, Accumulator->getOperand(0), VregClass);
+ auto [SrcReg0Sub0, SrcReg0Sub1] = ExtractSubRegs(
+ MI, Accumulator->getOperand(0), VregClass, ST, MRI);
BuildRegSequence(*ComputeLoop, I, AccumulatorVReg, SrcReg0Sub0,
SrcReg0Sub1);
BuildMI(*ComputeLoop, I, DL, TII->get(Opc), LaneMaskReg)
@@ -6209,8 +6213,8 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
BuildMI(*ComputeLoop, I, DL,
TII->get(AMDGPU::V_READFIRSTLANE_B32), LaneValHi);
MachineBasicBlock::iterator Iters = *ReadLaneLo;
- auto [Op1L, Op1H] =
- ExtractSubRegs(*Iters, DstVregInst->getOperand(0), VregRC);
+ auto [Op1L, Op1H] = ExtractSubRegs(*Iters, DstVregInst->getOperand(0),
+ VregRC, ST, MRI);
ReadLaneLo.addReg(Op1L);
ReadLaneHi.addReg(Op1H);
NewAccumulator =
More information about the llvm-commits
mailing list