[llvm] cee66b7 - [AMDGPU]Refactor `lowerWaveReduce` for maintainability (#189223)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 15 02:20:13 PDT 2026
Author: Aaditya
Date: 2026-04-15T14:50:07+05:30
New Revision: cee66b7a4bd7597061e880a3dd3a218167787710
URL: https://github.com/llvm/llvm-project/commit/cee66b7a4bd7597061e880a3dd3a218167787710
DIFF: https://github.com/llvm/llvm-project/commit/cee66b7a4bd7597061e880a3dd3a218167787710.diff
LOG: [AMDGPU]Refactor `lowerWaveReduce` for maintainability (#189223)
The function to lower wave reduce pseudos is already quite
large ,and there are yet a few more operations to support.
Refactoring some of the code to make it more manageable.
Summary of changes:
1. Moved the expansion for `V_CNDMASK_B64_PSEUDO` to a
separate function. It's needed for 64 bit dpp operations.
2. Collapsed `getIdentityValueFor32BitWaveReduction` and
`getIdentityValueFor64BitWaveReduction` into a single
function which returns a 64 bit unsigned value.
3. Modified `getDPPOpcForWaveReduction` to also return
the `Clamp` opcode.
4. Added a lambda: `BuildRegSequence` and a static function
`ExtractSubRegs` as those code blocks are repeated with
little variation.
5. Moved logic for setting identity value in inactive lanes
to `BuildSetInactiveInstr`.
Added:
Modified:
llvm/lib/Target/AMDGPU/SIISelLowering.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 6750dfcbaac62..9008db2296236 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -5510,7 +5510,7 @@ static MachineBasicBlock *emitIndirectDst(MachineInstr &MI,
return LoopBB;
}
-static MachineBasicBlock *Expand64BitScalarArithmetic(MachineInstr &MI,
+static MachineBasicBlock *expand64BitScalarArithmetic(MachineInstr &MI,
MachineBasicBlock *BB) {
// For targets older than GFX12, we emit a sequence of 32-bit operations.
// For GFX12, we emit s_add_u64 and s_sub_u64.
@@ -5561,7 +5561,71 @@ static MachineBasicBlock *Expand64BitScalarArithmetic(MachineInstr &MI,
return BB;
}
-static uint32_t getIdentityValueFor32BitWaveReduction(unsigned Opc) {
+static void expand64BitV_CNDMASK(MachineInstr &MI, MachineBasicBlock *BB) {
+ MachineFunction *MF = BB->getParent();
+ const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
+ const SIInstrInfo *TII = ST.getInstrInfo();
+ const SIRegisterInfo *TRI = ST.getRegisterInfo();
+ MachineRegisterInfo &MRI = MF->getRegInfo();
+ const DebugLoc &DL = MI.getDebugLoc();
+ Register Dst = MI.getOperand(0).getReg();
+ const MachineOperand &Src0 = MI.getOperand(1);
+ const MachineOperand &Src1 = MI.getOperand(2);
+ Register SrcCond = MI.getOperand(3).getReg();
+
+ Register DstLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ Register DstHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ const TargetRegisterClass *CondRC = TRI->getWaveMaskRegClass();
+ Register SrcCondCopy = MRI.createVirtualRegister(CondRC);
+
+ int Src0Idx =
+ AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0);
+ int Src1Idx =
+ AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src1);
+ const TargetRegisterClass *Src0RC =
+ TRI->getAllocatableClass(TII->getRegClass(MI.getDesc(), Src0Idx));
+ const TargetRegisterClass *Src1RC =
+ TRI->getAllocatableClass(TII->getRegClass(MI.getDesc(), Src1Idx));
+
+ const TargetRegisterClass *Src0SubRC =
+ TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
+ const TargetRegisterClass *Src1SubRC =
+ TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
+
+ MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(
+ MI, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC);
+ MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(
+ MI, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC);
+
+ MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(
+ MI, MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC);
+ MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
+ MI, MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC);
+
+ BuildMI(*BB, MI, DL, TII->get(AMDGPU::COPY), SrcCondCopy).addReg(SrcCond);
+ BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstLo)
+ .addImm(0)
+ .add(Src0Sub0)
+ .addImm(0)
+ .add(Src1Sub0)
+ .addReg(SrcCondCopy);
+
+ BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstHi)
+ .addImm(0)
+ .add(Src0Sub1)
+ .addImm(0)
+ .add(Src1Sub1)
+ .addReg(SrcCondCopy);
+
+ BuildMI(*BB, MI, DL, TII->get(AMDGPU::REG_SEQUENCE), Dst)
+ .addReg(DstLo)
+ .addImm(AMDGPU::sub0)
+ .addReg(DstHi)
+ .addImm(AMDGPU::sub1);
+ MI.eraseFromParent();
+}
+
+static uint64_t getIdentityValueForWaveReduction(unsigned Opc) {
switch (Opc) {
case AMDGPU::S_MIN_U32:
return std::numeric_limits<uint32_t>::max();
@@ -5584,15 +5648,7 @@ static uint32_t getIdentityValueFor32BitWaveReduction(unsigned Opc) {
return std::numeric_limits<uint32_t>::max();
case AMDGPU::V_MIN_F32_e64:
case AMDGPU::V_MAX_F32_e64:
- return 0x7fc00000; // qNAN
- default:
- llvm_unreachable(
- "Unexpected opcode in getIdentityValueFor32BitWaveReduction");
- }
-}
-
-static uint64_t getIdentityValueFor64BitWaveReduction(unsigned Opc) {
- switch (Opc) {
+ return 0x7fc00000; // qNAN
case AMDGPU::V_CMP_LT_U64_e64: // umin.u64
return std::numeric_limits<uint64_t>::max();
case AMDGPU::V_CMP_LT_I64_e64: // min.i64
@@ -5617,8 +5673,7 @@ static uint64_t getIdentityValueFor64BitWaveReduction(unsigned Opc) {
case AMDGPU::V_ADD_F64_pseudo_e64:
return 0x8000000000000000; // -0.0
default:
- llvm_unreachable(
- "Unexpected opcode in getIdentityValueFor64BitWaveReduction");
+ llvm_unreachable("Unexpected opcode in getIdentityValueForWaveReduction");
}
}
@@ -5640,37 +5695,72 @@ static bool isFloatingPointWaveReduceOperation(unsigned Opc) {
Opc == AMDGPU::V_ADD_F64_e64 || Opc == AMDGPU::V_ADD_F64_pseudo_e64;
}
-static unsigned getDPPOpcForWaveReduction(unsigned Opc,
- const GCNSubtarget &ST) {
+static std::tuple<unsigned, unsigned>
+getDPPOpcForWaveReduction(unsigned Opc, const GCNSubtarget &ST) {
+ unsigned DPPOpc;
switch (Opc) {
case AMDGPU::S_MIN_U32:
- return AMDGPU::V_MIN_U32_dpp;
+ DPPOpc = AMDGPU::V_MIN_U32_dpp;
+ break;
case AMDGPU::S_MIN_I32:
- return AMDGPU::V_MIN_I32_dpp;
+ DPPOpc = AMDGPU::V_MIN_I32_dpp;
+ break;
case AMDGPU::S_MAX_U32:
- return AMDGPU::V_MAX_U32_dpp;
+ DPPOpc = AMDGPU::V_MAX_U32_dpp;
+ break;
case AMDGPU::S_MAX_I32:
- return AMDGPU::V_MAX_I32_dpp;
+ DPPOpc = AMDGPU::V_MAX_I32_dpp;
+ break;
case AMDGPU::S_ADD_I32:
case AMDGPU::S_SUB_I32:
- return ST.hasAddNoCarryInsts() ? AMDGPU::V_ADD_U32_dpp
- : AMDGPU::V_ADD_CO_U32_dpp;
+ DPPOpc = ST.hasAddNoCarryInsts() ? AMDGPU::V_ADD_U32_dpp
+ : AMDGPU::V_ADD_CO_U32_dpp;
+ break;
case AMDGPU::S_AND_B32:
- return AMDGPU::V_AND_B32_dpp;
+ DPPOpc = AMDGPU::V_AND_B32_dpp;
+ break;
case AMDGPU::S_OR_B32:
- return AMDGPU::V_OR_B32_dpp;
+ DPPOpc = AMDGPU::V_OR_B32_dpp;
+ break;
case AMDGPU::S_XOR_B32:
- return AMDGPU::V_XOR_B32_dpp;
+ DPPOpc = AMDGPU::V_XOR_B32_dpp;
+ break;
case AMDGPU::V_ADD_F32_e64:
case AMDGPU::V_SUB_F32_e64:
- return AMDGPU::V_ADD_F32_dpp;
+ DPPOpc = AMDGPU::V_ADD_F32_dpp;
+ break;
case AMDGPU::V_MIN_F32_e64:
- return AMDGPU::V_MIN_F32_dpp;
+ DPPOpc = AMDGPU::V_MIN_F32_dpp;
+ break;
case AMDGPU::V_MAX_F32_e64:
- return AMDGPU::V_MAX_F32_dpp;
+ DPPOpc = AMDGPU::V_MAX_F32_dpp;
+ break;
default:
llvm_unreachable("unhandled lane op");
}
+ bool isFPOp = isFloatingPointWaveReduceOperation(Opc);
+ unsigned ClampOpc = Opc;
+ if (!isFPOp) {
+ if (Opc == AMDGPU::S_SUB_I32)
+ ClampOpc = AMDGPU::S_ADD_I32;
+ ClampOpc = ST.getInstrInfo()->getVALUOp(ClampOpc);
+ }
+ return {DPPOpc, ClampOpc};
+}
+
+static std::pair<Register, Register>
+ExtractSubRegs(MachineInstr &MI, MachineOperand &Op,
+ const TargetRegisterClass *SrcRC, const GCNSubtarget &ST,
+ MachineRegisterInfo &MRI) {
+ const SIRegisterInfo *TRI = ST.getRegisterInfo();
+ const SIInstrInfo *TII = ST.getInstrInfo();
+ const TargetRegisterClass *SrcSubRC =
+ TRI->getSubRegisterClass(SrcRC, AMDGPU::sub0);
+ Register Op1L =
+ TII->buildExtractSubReg(MI, MRI, Op, SrcRC, AMDGPU::sub0, SrcSubRC);
+ Register Op1H =
+ TII->buildExtractSubReg(MI, MRI, Op, SrcRC, AMDGPU::sub1, SrcSubRC);
+ return {Op1L, Op1H};
}
static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
@@ -5689,6 +5779,17 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
unsigned Stratergy = static_cast<unsigned>(MI.getOperand(2).getImm());
enum WAVE_REDUCE_STRATEGY : unsigned { DEFAULT = 0, ITERATIVE = 1, DPP = 2 };
MachineBasicBlock *RetBB = nullptr;
+ auto BuildRegSequence = [&](MachineBasicBlock &BB,
+ MachineBasicBlock::iterator MI, Register Dst,
+ Register Src0, Register Src1) {
+ auto RegSequence =
+ BuildMI(BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), Dst)
+ .addReg(Src0)
+ .addImm(AMDGPU::sub0)
+ .addReg(Src1)
+ .addImm(AMDGPU::sub1);
+ return RegSequence;
+ };
if (isSGPR) {
switch (Opc) {
case AMDGPU::S_MIN_U32:
@@ -5770,29 +5871,15 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
Register DestSub1 =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
-
- const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg);
- const TargetRegisterClass *SrcSubRC =
- TRI->getSubRegisterClass(SrcRC, AMDGPU::sub0);
-
- MachineOperand Op1L = TII->buildExtractSubRegOrImm(
- MI, MRI, MI.getOperand(1), SrcRC, AMDGPU::sub0, SrcSubRC);
- MachineOperand Op1H = TII->buildExtractSubRegOrImm(
- MI, MRI, MI.getOperand(1), SrcRC, AMDGPU::sub1, SrcSubRC);
-
+ auto [Op1L, Op1H] = ExtractSubRegs(MI, MI.getOperand(1),
+ MRI.getRegClass(SrcReg), ST, MRI);
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DestSub0)
- .add(Op1L)
+ .addReg(Op1L)
.addReg(ParityRegister);
-
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DestSub1)
- .add(Op1H)
+ .addReg(Op1H)
.addReg(ParityRegister);
-
- BuildMI(BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), DstReg)
- .addReg(DestSub0)
- .addImm(AMDGPU::sub0)
- .addReg(DestSub1)
- .addImm(AMDGPU::sub1);
+ BuildRegSequence(BB, MI, DstReg, DestSub0, DestSub1);
}
break;
}
@@ -5828,16 +5915,8 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
Register NegatedValHi =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
-
- const TargetRegisterClass *Src1RC = MRI.getRegClass(SrcReg);
- const TargetRegisterClass *Src1SubRC =
- TRI->getSubRegisterClass(Src1RC, AMDGPU::sub0);
-
- MachineOperand Op1L = TII->buildExtractSubRegOrImm(
- MI, MRI, MI.getOperand(1), Src1RC, AMDGPU::sub0, Src1SubRC);
- MachineOperand Op1H = TII->buildExtractSubRegOrImm(
- MI, MRI, MI.getOperand(1), Src1RC, AMDGPU::sub1, Src1SubRC);
-
+ auto [Op1L, Op1H] = ExtractSubRegs(MI, MI.getOperand(1),
+ MRI.getRegClass(SrcReg), ST, MRI);
if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_SUB_I32), NegatedValLo)
.addImm(0)
@@ -5848,20 +5927,20 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
.addImm(31)
.setOperandDead(3); // Dead scc
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), Op1L_Op0H_Reg)
- .add(Op1L)
+ .addReg(Op1L)
.addReg(NegatedValHi);
}
Register LowOpcode = Opc == AMDGPU::S_SUB_U64_PSEUDO
? NegatedValLo
: NewAccumulator->getOperand(0).getReg();
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DestSub0)
- .add(Op1L)
+ .addReg(Op1L)
.addReg(LowOpcode);
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_HI_U32), CarryReg)
- .add(Op1L)
+ .addReg(Op1L)
.addReg(LowOpcode);
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), Op1H_Op0L_Reg)
- .add(Op1H)
+ .addReg(Op1H)
.addReg(LowOpcode);
Register HiVal = Opc == AMDGPU::S_SUB_U64_PSEUDO ? AddReg : DestSub1;
@@ -5876,11 +5955,7 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
.addReg(Op1L_Op0H_Reg)
.setOperandDead(3); // Dead scc
}
- BuildMI(BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), DstReg)
- .addReg(DestSub0)
- .addImm(AMDGPU::sub0)
- .addReg(DestSub1)
- .addImm(AMDGPU::sub1);
+ BuildRegSequence(BB, MI, DstReg, DestSub0, DestSub1);
break;
}
case AMDGPU::V_ADD_F32_e64:
@@ -5926,27 +6001,17 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
Register LaneValueHiReg =
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
- const TargetRegisterClass *VregSubRC =
- TRI->getSubRegisterClass(VregRC, AMDGPU::sub0);
- MachineOperand Op1L =
- TII->buildExtractSubRegOrImm(MI, MRI, DestVregInst->getOperand(0),
- VregRC, AMDGPU::sub0, VregSubRC);
- MachineOperand Op1H =
- TII->buildExtractSubRegOrImm(MI, MRI, DestVregInst->getOperand(0),
- VregRC, AMDGPU::sub1, VregSubRC);
+ auto [Op1L, Op1H] =
+ ExtractSubRegs(MI, DestVregInst->getOperand(0), VregRC, ST, MRI);
// lane value input should be in an sgpr
BuildMI(BB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
LaneValueLoReg)
- .add(Op1L);
+ .addReg(Op1L);
BuildMI(BB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
LaneValueHiReg)
- .add(Op1H);
+ .addReg(Op1H);
NewAccumulator =
- BuildMI(BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), DstReg)
- .addReg(LaneValueLoReg)
- .addImm(AMDGPU::sub0)
- .addReg(LaneValueHiReg)
- .addImm(AMDGPU::sub1);
+ BuildRegSequence(BB, MI, DstReg, LaneValueLoReg, LaneValueHiReg);
}
}
}
@@ -5990,19 +6055,15 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
// Create initial values of induction variable from Exec, Accumulator and
// insert branch instr to newly created ComputeBlock
BuildMI(BB, I, DL, TII->get(MovOpcForExec), LoopIterator).addReg(ExecReg);
- if (is32BitOpc) {
- uint32_t IdentityValue = getIdentityValueFor32BitWaveReduction(Opc);
- BuildMI(BB, I, DL, TII->get(AMDGPU::S_MOV_B32), IdentityValReg)
- .addImm(IdentityValue);
- } else {
- uint64_t IdentityValue =
- MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64
- ? 0x0 // +0.0 for double sub reduction
- : getIdentityValueFor64BitWaveReduction(Opc);
- BuildMI(BB, I, DL, TII->get(AMDGPU::S_MOV_B64_IMM_PSEUDO),
- IdentityValReg)
- .addImm(IdentityValue);
- }
+ uint64_t IdentityValue =
+ MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64
+ ? 0x0 // +0.0 for double sub reduction
+ : getIdentityValueForWaveReduction(Opc);
+ BuildMI(BB, I, DL,
+ TII->get(is32BitOpc ? AMDGPU::S_MOV_B32
+ : AMDGPU::S_MOV_B64_IMM_PSEUDO),
+ IdentityValReg)
+ .addImm(IdentityValue);
// clang-format off
BuildMI(BB, I, DL, TII->get(AMDGPU::S_BRANCH))
.addMBB(ComputeLoop);
@@ -6062,29 +6123,19 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
Register LaneValReg =
MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
- const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg);
- const TargetRegisterClass *SrcSubRC =
- TRI->getSubRegisterClass(SrcRC, AMDGPU::sub0);
- MachineOperand Op1L = TII->buildExtractSubRegOrImm(
- MI, MRI, MI.getOperand(1), SrcRC, AMDGPU::sub0, SrcSubRC);
- MachineOperand Op1H = TII->buildExtractSubRegOrImm(
- MI, MRI, MI.getOperand(1), SrcRC, AMDGPU::sub1, SrcSubRC);
+ auto [Op1L, Op1H] = ExtractSubRegs(MI, MI.getOperand(1),
+ MRI.getRegClass(SrcReg), ST, MRI);
// lane value input should be in an sgpr
BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::V_READLANE_B32),
LaneValueLoReg)
- .add(Op1L)
+ .addReg(Op1L)
.addReg(FF1Reg);
BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::V_READLANE_B32),
LaneValueHiReg)
- .add(Op1H)
+ .addReg(Op1H)
.addReg(FF1Reg);
- auto LaneValue =
- BuildMI(*ComputeLoop, I, DL, TII->get(TargetOpcode::REG_SEQUENCE),
- LaneValReg)
- .addReg(LaneValueLoReg)
- .addImm(AMDGPU::sub0)
- .addReg(LaneValueHiReg)
- .addImm(AMDGPU::sub1);
+ auto LaneValue = BuildRegSequence(*ComputeLoop, I, LaneValReg,
+ LaneValueLoReg, LaneValueHiReg);
switch (Opc) {
case AMDGPU::S_OR_B64:
case AMDGPU::S_AND_B64:
@@ -6106,21 +6157,11 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src);
const TargetRegisterClass *VregClass =
TRI->getAllocatableClass(TII->getRegClass(MI.getDesc(), SrcIdx));
- const TargetRegisterClass *VSubRegClass =
- TRI->getSubRegisterClass(VregClass, AMDGPU::sub0);
Register AccumulatorVReg = MRI.createVirtualRegister(VregClass);
- MachineOperand SrcReg0Sub0 = TII->buildExtractSubRegOrImm(
- MI, MRI, Accumulator->getOperand(0), VregClass, AMDGPU::sub0,
- VSubRegClass);
- MachineOperand SrcReg0Sub1 = TII->buildExtractSubRegOrImm(
- MI, MRI, Accumulator->getOperand(0), VregClass, AMDGPU::sub1,
- VSubRegClass);
- BuildMI(*ComputeLoop, I, DL, TII->get(TargetOpcode::REG_SEQUENCE),
- AccumulatorVReg)
- .add(SrcReg0Sub0)
- .addImm(AMDGPU::sub0)
- .add(SrcReg0Sub1)
- .addImm(AMDGPU::sub1);
+ auto [SrcReg0Sub0, SrcReg0Sub1] = ExtractSubRegs(
+ MI, Accumulator->getOperand(0), VregClass, ST, MRI);
+ BuildRegSequence(*ComputeLoop, I, AccumulatorVReg, SrcReg0Sub0,
+ SrcReg0Sub1);
BuildMI(*ComputeLoop, I, DL, TII->get(Opc), LaneMaskReg)
.addReg(LaneValue->getOperand(0).getReg())
.addReg(AccumulatorVReg);
@@ -6146,8 +6187,6 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src);
const TargetRegisterClass *VregRC =
TRI->getAllocatableClass(TII->getRegClass(MI.getDesc(), SrcIdx));
- const TargetRegisterClass *VregSubRC =
- TRI->getSubRegisterClass(VregRC, AMDGPU::sub0);
Register AccumulatorVReg = MRI.createVirtualRegister(VregRC);
Register DstVreg = MRI.createVirtualRegister(VregRC);
Register LaneValLo =
@@ -6175,20 +6214,12 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
BuildMI(*ComputeLoop, I, DL,
TII->get(AMDGPU::V_READFIRSTLANE_B32), LaneValHi);
MachineBasicBlock::iterator Iters = *ReadLaneLo;
- MachineOperand Op1L = TII->buildExtractSubRegOrImm(
- Iters, MRI, DstVregInst->getOperand(0), VregRC, AMDGPU::sub0,
- VregSubRC);
- MachineOperand Op1H = TII->buildExtractSubRegOrImm(
- Iters, MRI, DstVregInst->getOperand(0), VregRC, AMDGPU::sub1,
- VregSubRC);
- ReadLaneLo.add(Op1L);
- ReadLaneHi.add(Op1H);
- NewAccumulator = BuildMI(*ComputeLoop, I, DL,
- TII->get(TargetOpcode::REG_SEQUENCE), DstReg)
- .addReg(LaneValLo)
- .addImm(AMDGPU::sub0)
- .addReg(LaneValHi)
- .addImm(AMDGPU::sub1);
+ auto [Op1L, Op1H] = ExtractSubRegs(*Iters, DstVregInst->getOperand(0),
+ VregRC, ST, MRI);
+ ReadLaneLo.addReg(Op1L);
+ ReadLaneHi.addReg(Op1H);
+ NewAccumulator =
+ BuildRegSequence(*ComputeLoop, I, DstReg, LaneValLo, LaneValHi);
break;
}
case AMDGPU::S_ADD_U64_PSEUDO:
@@ -6197,7 +6228,7 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
.addReg(Accumulator->getOperand(0).getReg())
.addReg(LaneValue->getOperand(0).getReg());
ComputeLoop =
- Expand64BitScalarArithmetic(*NewAccumulator, ComputeLoop);
+ expand64BitScalarArithmetic(*NewAccumulator, ComputeLoop);
break;
}
}
@@ -6225,7 +6256,6 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
} else {
assert(ST.hasDPP() && "Sub Target does not support DPP Operations");
- bool IsFPOp = isFloatingPointWaveReduceOperation(Opc);
Register SrcWithIdentity = MRI.createVirtualRegister(SrcRegClass);
Register IdentityVGPR = MRI.createVirtualRegister(SrcRegClass);
Register IdentitySGPR = MRI.createVirtualRegister(DstRegClass);
@@ -6241,29 +6271,32 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
Register FinalDPPResult;
BuildMI(BB, MI, DL, TII->get(AMDGPU::IMPLICIT_DEF), UndefExec);
- uint32_t IdentityValue = getIdentityValueFor32BitWaveReduction(Opc);
+ uint64_t IdentityValue = getIdentityValueForWaveReduction(Opc);
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B32), IdentitySGPR)
.addImm(IdentityValue);
BuildMI(BB, MI, DL, TII->get(AMDGPU::COPY), IdentityVGPR)
.addReg(IdentitySGPR);
-
- // Set inactive lanes to the identity value.
- BuildMI(BB, MI, DL, TII->get(AMDGPU::V_SET_INACTIVE_B32), SrcWithIdentity)
- .addImm(0) // src0 modifiers
- .addReg(SrcReg) // src0
- .addImm(0) // src1 modifiers
- .addReg(IdentityVGPR) // identity value for inactive lanes
- .addReg(UndefExec); // bool i1
-
- unsigned DPPOpc = getDPPOpcForWaveReduction(Opc, ST);
+ auto DPPClampOpcPair = getDPPOpcForWaveReduction(Opc, ST);
+ unsigned DPPOpc = std::get<0>(DPPClampOpcPair);
+ unsigned ClampOpc = std::get<1>(DPPClampOpcPair);
+ auto BuildSetInactiveInstr = [&](Register Dst, Register Src0,
+ Register Src1) {
+ return BuildMI(BB, MI, DL, TII->get(AMDGPU::V_SET_INACTIVE_B32),
+ Dst)
+ .addImm(0) // src0 modifiers
+ .addReg(Src0) // src0
+ .addImm(0) // src1 modifiers
+ .addReg(Src1) // identity value for inactive lanes
+ .addReg(UndefExec); // bool i1
+ };
auto BuildDPPMachineInstr = [&](Register Dst, Register Src,
unsigned DPPCtrl) {
auto DPPInstr =
BuildMI(BB, MI, DL, TII->get(DPPOpc), Dst).addReg(Src); // old
- if (IsFPOp)
+ if (isFPOp)
DPPInstr.addImm(SISrcMods::NONE); // src0 modifier
DPPInstr.addReg(Src); // src0
- if (IsFPOp)
+ if (isFPOp)
DPPInstr.addImm(SISrcMods::NONE); // src1 modifier
DPPInstr
.addReg(Src) // src1
@@ -6273,24 +6306,22 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
.addImm(0); // bound-control
};
auto BuildClampInstr = [&](Register Dst, Register Src0, Register Src1) {
- unsigned ClampOpc = Opc;
- if (!IsFPOp) {
- if (Opc == AMDGPU::S_SUB_I32)
- ClampOpc = AMDGPU::S_ADD_I32;
- ClampOpc = TII->getVALUOp(ClampOpc);
- }
auto ClampInstr = BuildMI(BB, MI, DL, TII->get(ClampOpc), Dst);
- if (IsFPOp)
+ if (isFPOp)
ClampInstr.addImm(SISrcMods::NONE); // src0 mod
ClampInstr.addReg(Src0); // src0
- if (IsFPOp)
+ if (isFPOp)
ClampInstr.addImm(SISrcMods::NONE); // src1 mod
ClampInstr.addReg(Src1); // src1
if (TII->hasIntClamp(*ClampInstr) || TII->hasFPClamp(*ClampInstr))
ClampInstr.addImm(0); // clamp
- if (IsFPOp)
+ if (isFPOp)
ClampInstr.addImm(0); // omod
};
+
+ // Set inactive lanes to the identity value.
+ BuildSetInactiveInstr(SrcWithIdentity, SrcReg, IdentityVGPR);
+
// DPP reduction
BuildDPPMachineInstr(DPPRowShr1, SrcWithIdentity,
AMDGPU::DPP::ROW_SHR_FIRST);
@@ -6504,7 +6535,7 @@ SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
}
case AMDGPU::S_ADD_U64_PSEUDO:
case AMDGPU::S_SUB_U64_PSEUDO: {
- return Expand64BitScalarArithmetic(MI, BB);
+ return expand64BitScalarArithmetic(MI, BB);
}
case AMDGPU::V_ADD_U64_PSEUDO:
case AMDGPU::V_SUB_U64_PSEUDO: {
@@ -6768,58 +6799,7 @@ SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
case AMDGPU::SI_KILL_I1_PSEUDO:
return splitKillBlock(MI, BB);
case AMDGPU::V_CNDMASK_B64_PSEUDO: {
- Register Dst = MI.getOperand(0).getReg();
- const MachineOperand &Src0 = MI.getOperand(1);
- const MachineOperand &Src1 = MI.getOperand(2);
- Register SrcCond = MI.getOperand(3).getReg();
-
- Register DstLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
- Register DstHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
- const auto *CondRC = TRI->getWaveMaskRegClass();
- Register SrcCondCopy = MRI.createVirtualRegister(CondRC);
-
- const TargetRegisterClass *Src0RC = Src0.isReg()
- ? MRI.getRegClass(Src0.getReg())
- : &AMDGPU::VReg_64RegClass;
- const TargetRegisterClass *Src1RC = Src1.isReg()
- ? MRI.getRegClass(Src1.getReg())
- : &AMDGPU::VReg_64RegClass;
-
- const TargetRegisterClass *Src0SubRC =
- TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
- const TargetRegisterClass *Src1SubRC =
- TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
-
- MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(
- MI, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC);
- MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(
- MI, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC);
-
- MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(
- MI, MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC);
- MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
- MI, MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC);
-
- BuildMI(*BB, MI, DL, TII->get(AMDGPU::COPY), SrcCondCopy).addReg(SrcCond);
- BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstLo)
- .addImm(0)
- .add(Src0Sub0)
- .addImm(0)
- .add(Src1Sub0)
- .addReg(SrcCondCopy);
- BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstHi)
- .addImm(0)
- .add(Src0Sub1)
- .addImm(0)
- .add(Src1Sub1)
- .addReg(SrcCondCopy);
-
- BuildMI(*BB, MI, DL, TII->get(AMDGPU::REG_SEQUENCE), Dst)
- .addReg(DstLo)
- .addImm(AMDGPU::sub0)
- .addReg(DstHi)
- .addImm(AMDGPU::sub1);
- MI.eraseFromParent();
+ expand64BitV_CNDMASK(MI, BB);
return BB;
}
case AMDGPU::SI_BR_UNDEF: {
More information about the llvm-commits
mailing list