[llvm] [AMDGPU] Waterfall loop codegen improvement in SIInstrInfo (PR #192415)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 30 01:06:15 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: gretay-amd
<details>
<summary>Changes</summary>
When generating waterfall loops, use the instructions `v_cmpx_eq_*` and `s_andn2_wrexec_*` as recommended for recent architectures, instead of `v_cmp_eq_*` and `s_and saveexec`.
This PR only updates waterfall loop code generation in `SIInstrInfo.cpp`. Other places that generated waterfall loops can be handled separately.
- Add new lane mask constants for `s_andn2_wrexec` and `v_cmpx_eq_u32/u64`
- Fix test `mubuf-legalize-operands.mir` to track liveness needed for verifying phi nodes
- Update .ll and .mir tests to accept the new instruction sequences
---
Patch is 645.04 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/192415.diff
22 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h (+9)
- (modified) llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp (+6)
- (modified) llvm/lib/Target/AMDGPU/SIInstrInfo.cpp (+96-49)
- (modified) llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll (+637-671)
- (modified) llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll (+673-712)
- (modified) llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll (+673-712)
- (modified) llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll (+33-36)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll (+36-38)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll (+67-69)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll (+38-36)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll (+72-74)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll (+72-74)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll (+108-114)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll (+108-114)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll (+39-41)
- (modified) llvm/test/CodeGen/AMDGPU/move-to-valu-vimage-vsample.ll (+38-46)
- (modified) llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll (+272-284)
- (modified) llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll (+272-284)
- (modified) llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir (+320-121)
- (modified) llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll (+26-27)
- (modified) llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll (+47-51)
- (modified) llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll (+16-12)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h b/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
index 3fdd962dcc30c..cac303dd22599 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
@@ -27,10 +27,13 @@ class LaneMaskConstants {
const unsigned AndN2Opc;
const unsigned AndN2SaveExecOpc;
const unsigned AndN2TermOpc;
+ const unsigned AndN2WRExecOpc;
const unsigned AndSaveExecOpc;
const unsigned AndSaveExecTermOpc;
const unsigned BfmOpc;
const unsigned CMovOpc;
+ const unsigned CmpXEqU32Opc;
+ const unsigned CmpXEqU64Opc;
const unsigned CSelectOpc;
const unsigned MovOpc;
const unsigned MovTermOpc;
@@ -52,12 +55,18 @@ class LaneMaskConstants {
: AMDGPU::S_ANDN2_SAVEEXEC_B64),
AndN2TermOpc(IsWave32 ? AMDGPU::S_ANDN2_B32_term
: AMDGPU::S_ANDN2_B64_term),
+ AndN2WRExecOpc(IsWave32 ? AMDGPU::S_ANDN2_WREXEC_B32
+ : AMDGPU::S_ANDN2_WREXEC_B64),
AndSaveExecOpc(IsWave32 ? AMDGPU::S_AND_SAVEEXEC_B32
: AMDGPU::S_AND_SAVEEXEC_B64),
AndSaveExecTermOpc(IsWave32 ? AMDGPU::S_AND_SAVEEXEC_B32_term
: AMDGPU::S_AND_SAVEEXEC_B64_term),
BfmOpc(IsWave32 ? AMDGPU::S_BFM_B32 : AMDGPU::S_BFM_B64),
CMovOpc(IsWave32 ? AMDGPU::S_CMOV_B32 : AMDGPU::S_CMOV_B64),
+ CmpXEqU32Opc(IsWave32 ? AMDGPU::V_CMPX_EQ_U32_nosdst_e32
+ : AMDGPU::V_CMPX_EQ_U32_nosdst_e64),
+ CmpXEqU64Opc(IsWave32 ? AMDGPU::V_CMPX_EQ_U64_nosdst_e32
+ : AMDGPU::V_CMPX_EQ_U64_nosdst_e64),
CSelectOpc(IsWave32 ? AMDGPU::S_CSELECT_B32 : AMDGPU::S_CSELECT_B64),
MovOpc(IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64),
MovTermOpc(IsWave32 ? AMDGPU::S_MOV_B32_term : AMDGPU::S_MOV_B64_term),
diff --git a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
index 5994eeb54095b..5cd6c847d0bbd 100644
--- a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
@@ -182,6 +182,12 @@ class SIFixSGPRCopiesLegacy : public MachineFunctionPass {
AU.setPreservesCFG();
MachineFunctionPass::getAnalysisUsage(AU);
}
+
+ // Waterfall expansion may introduce Phi nodes and -verify-machineinstrs will
+ // fail.
+ MachineFunctionProperties getClearedProperties() const override {
+ return MachineFunctionProperties().setNoPHIs();
+ }
};
} // end anonymous namespace
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index bc3052b139d18..1128566cb36ce 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7092,22 +7092,43 @@ void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB,
// Emit the actual waterfall loop, executing the wrapped instruction for each
// unique value of \p ScalarOps across all lanes. In the best case we execute 1
// iteration, in the worst case we execute 64 (once per lane).
-static void
-emitLoadScalarOpsFromVGPRLoop(const SIInstrInfo &TII,
- MachineRegisterInfo &MRI,
- MachineBasicBlock &LoopBB,
- MachineBasicBlock &BodyBB,
- const DebugLoc &DL,
- ArrayRef<MachineOperand *> ScalarOps) {
+static void emitLoadScalarOpsFromVGPRLoop(
+ const SIInstrInfo &TII, MachineRegisterInfo &MRI, MachineBasicBlock &PredBB,
+ MachineBasicBlock &LoopBB, MachineBasicBlock &BodyBB, const DebugLoc &DL,
+ ArrayRef<MachineOperand *> ScalarOps) {
MachineFunction &MF = *LoopBB.getParent();
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIRegisterInfo *TRI = ST.getRegisterInfo();
const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
+ // Emit [v_cmpx_eq] and [s_andn2_wrexec] when these instructions are
+ // available.
+ // Otherwise, use the previous pattern of [v_cmp_eq], [s_and_saveexec],
+ // and [s_xor].
+ // TODO: Accurately detect the availability of [s_andn2_wrexec] instruction
+ // in the target. For now, use the same condition as for the detection
+ // [v_cmpx_eq].
+ bool UseNewExecInstructions = ST.hasNoSdstCMPX();
+
MachineBasicBlock::iterator I = LoopBB.begin();
Register CondReg;
+ Register PhiExec = MRI.createVirtualRegister(BoolXExecRC);
+ Register NewExec = MRI.createVirtualRegister(BoolXExecRC);
+
+ if (UseNewExecInstructions) {
+ Register InitExec = MRI.createVirtualRegister(BoolXExecRC);
+ BuildMI(PredBB, PredBB.end(), DL, TII.get(LMC.MovOpc), InitExec)
+ .addReg(LMC.ExecReg);
+
+ BuildMI(LoopBB, I, DL, TII.get(TargetOpcode::PHI), PhiExec)
+ .addReg(InitExec)
+ .addMBB(&PredBB)
+ .addReg(NewExec)
+ .addMBB(&BodyBB);
+ }
+
for (MachineOperand *ScalarOp : ScalarOps) {
unsigned RegSize = TRI->getRegSizeInBits(ScalarOp->getReg(), MRI);
unsigned NumSubRegs = RegSize / 32;
@@ -7119,21 +7140,27 @@ emitLoadScalarOpsFromVGPRLoop(const SIInstrInfo &TII,
BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurReg)
.addReg(VScalarOp);
- Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
+ if (UseNewExecInstructions) {
+ BuildMI(LoopBB, I, DL, TII.get(LMC.CmpXEqU32Opc))
+ .addReg(CurReg)
+ .addReg(VScalarOp);
+ } else {
+ Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
- BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U32_e64), NewCondReg)
- .addReg(CurReg)
- .addReg(VScalarOp);
+ BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U32_e64), NewCondReg)
+ .addReg(CurReg)
+ .addReg(VScalarOp);
- // Combine the comparison results with AND.
- if (!CondReg) // First.
- CondReg = NewCondReg;
- else { // If not the first, we create an AND.
- Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
- BuildMI(LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)
- .addReg(CondReg)
- .addReg(NewCondReg);
- CondReg = AndReg;
+ // Combine the comparison results with AND.
+ if (!CondReg) // First.
+ CondReg = NewCondReg;
+ else { // If not the first, we create an AND.
+ Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
+ BuildMI(LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)
+ .addReg(CondReg)
+ .addReg(NewCondReg);
+ CondReg = AndReg;
+ }
}
// Update ScalarOp operand to use the SGPR ScalarOp.
@@ -7171,25 +7198,36 @@ emitLoadScalarOpsFromVGPRLoop(const SIInstrInfo &TII,
.addReg(CurRegHi)
.addImm(AMDGPU::sub1);
- Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
- auto Cmp = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64),
- NewCondReg)
- .addReg(CurReg);
- if (NumSubRegs <= 2)
- Cmp.addReg(VScalarOp);
- else
- Cmp.addReg(VScalarOp, VScalarOpUndef,
- TRI->getSubRegFromChannel(Idx, 2));
-
- // Combine the comparison results with AND.
- if (!CondReg) // First.
- CondReg = NewCondReg;
- else { // If not the first, we create an AND.
- Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
- BuildMI(LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)
- .addReg(CondReg)
- .addReg(NewCondReg);
- CondReg = AndReg;
+ if (UseNewExecInstructions) {
+ auto CmpX =
+ BuildMI(LoopBB, I, DL, TII.get(LMC.CmpXEqU64Opc)).addReg(CurReg);
+
+ if (NumSubRegs <= 2)
+ CmpX.addReg(VScalarOp);
+ else
+ CmpX.addReg(VScalarOp, VScalarOpUndef,
+ TRI->getSubRegFromChannel(Idx, 2));
+ } else {
+ Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
+ auto Cmp = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64),
+ NewCondReg)
+ .addReg(CurReg);
+ if (NumSubRegs <= 2)
+ Cmp.addReg(VScalarOp);
+ else
+ Cmp.addReg(VScalarOp, VScalarOpUndef,
+ TRI->getSubRegFromChannel(Idx, 2));
+
+ // Combine the comparison results with AND.
+ if (!CondReg) // First.
+ CondReg = NewCondReg;
+ else { // If not the first, we create an AND.
+ Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
+ BuildMI(LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)
+ .addReg(CondReg)
+ .addReg(NewCondReg);
+ CondReg = AndReg;
+ }
}
} // End for loop.
@@ -7211,20 +7249,29 @@ emitLoadScalarOpsFromVGPRLoop(const SIInstrInfo &TII,
}
}
- Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
- MRI.setSimpleHint(SaveExec, CondReg);
+ Register SaveExec;
+ if (!UseNewExecInstructions) {
+ SaveExec = MRI.createVirtualRegister(BoolXExecRC);
+ MRI.setSimpleHint(SaveExec, CondReg);
- // Update EXEC to matching lanes, saving original to SaveExec.
- BuildMI(LoopBB, I, DL, TII.get(LMC.AndSaveExecOpc), SaveExec)
- .addReg(CondReg, RegState::Kill);
+ // Update EXEC to matching lanes, saving original to SaveExec.
+ BuildMI(LoopBB, I, DL, TII.get(LMC.AndSaveExecOpc), SaveExec)
+ .addReg(CondReg, RegState::Kill);
+ }
// The original instruction is here; we insert the terminators after it.
I = BodyBB.end();
- // Update EXEC, switch all done bits to 0 and all todo bits to 1.
- BuildMI(BodyBB, I, DL, TII.get(LMC.XorTermOpc), LMC.ExecReg)
- .addReg(LMC.ExecReg)
- .addReg(SaveExec);
+ if (UseNewExecInstructions) {
+ MRI.setSimpleHint(NewExec, PhiExec);
+ BuildMI(BodyBB, I, DL, TII.get(LMC.AndN2WRExecOpc), NewExec)
+ .addReg(PhiExec);
+ } else {
+ // Update EXEC, switch all done bits to 0 and all todo bits to 1.
+ BuildMI(BodyBB, I, DL, TII.get(LMC.XorTermOpc), LMC.ExecReg)
+ .addReg(LMC.ExecReg)
+ .addReg(SaveExec);
+ }
BuildMI(BodyBB, I, DL, TII.get(AMDGPU::SI_WATERFALL_LOOP)).addMBB(&LoopBB);
}
@@ -7322,7 +7369,7 @@ loadScalarOperandsFromVGPR(const SIInstrInfo &TII, MachineInstr &MI,
}
}
- emitLoadScalarOpsFromVGPRLoop(TII, MRI, *LoopBB, *BodyBB, DL, ScalarOps);
+ emitLoadScalarOpsFromVGPRLoop(TII, MRI, MBB, *LoopBB, *BodyBB, DL, ScalarOps);
MachineBasicBlock::iterator First = RemainderBB->begin();
// Restore SCC
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index b5844049fd287..f27ef068911c8 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -372,31 +372,32 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_add_f32 v5, v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_add_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB2_1
; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, v5
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
@@ -431,27 +432,25 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s1, exec_lo
+; GFX11-NEXT: s_mov_b32 s4, exec_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_mov_b32 s5, s4
; GFX11-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_add_f32 v5, v4, s[4:7], 0 offen offset:1024 glc
+; GFX11-NEXT: buffer_atomic_add_f32 v5, v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-NEXT: ; implicit-def: $vgpr4
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB2_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-NEXT: s_mov_b32 exec_lo, s4
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v0, v5
; GFX11-NEXT: buffer_gl1_inv
@@ -461,60 +460,60 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v8, v4, s[8:11], 0 offen offset:1024
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v8, v4, s[4:7], 0 offen offset:1024
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB2_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: .LBB2_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB2_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_add_f32_e32 v7, v8, v5
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_mov_b32_e32 v6, v7
; GFX10-NEXT: v_mov_b32_e32 v7, v8
; GFX10-NEXT: .LBB2_4: ; Parent Loop BB2_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB2_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX10-NEXT: v_mov_b32_e32 v8, v6
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB2_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_mov_b32_e32 v0, v6
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -2310,66 +2309,66 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s5, s4
; GF...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/192415
More information about the llvm-commits
mailing list