[llvm] [AMDGPU] Sink async DMA out of s_cbranch_execz then-blocks (PR #196374)
Vigneshwar Jayakumar via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 6 02:25:06 PDT 2026
https://github.com/VigneshwarJ updated https://github.com/llvm/llvm-project/pull/196374
>From 530ccba92463d683aad675615035f0a7a9b5e6ec Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Thu, 7 May 2026 12:12:30 -0500
Subject: [PATCH 1/3] i[AMDGPU] Sink async DMA out of s_cbranch_execz
then-blocks
LLVM lowers a divergent branch around global_load_async_to_lds /
global_store_async_from_lds with s_cbranch_execz, fully-masked waves
skip the DMA entirely, forcing software-pipelined kernels to use
conservative waitcnts.
Sink each async DMA into the join block, immediately before the
SI_END_CF EXEC restore. EXEC at the sunk slot is the same masked value,
so per-lane behavior is unchanged, but async_waitcnts can be pipelined.
---
llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp | 136 +++++++++++
.../AMDGPU/sink-async-dma-out-of-execz.ll | 216 ++++++++++++++++++
2 files changed, 352 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/sink-async-dma-out-of-execz.ll
diff --git a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
index 9cc86e84407b1..1efc15233f080 100644
--- a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
@@ -70,6 +70,11 @@ static cl::opt<bool>
RemoveRedundantEndcf("amdgpu-remove-redundant-endcf",
cl::init(true), cl::ReallyHidden);
+static cl::opt<bool> EnableSinkAsyncDMAOutOfExecz(
+ "amdgpu-sink-async-dma-out-of-execz", cl::init(true), cl::ReallyHidden,
+ cl::desc("Sink async DMA out of EXECZ-skipped then-blocks for "
+ "ASYNCcnt determinism"));
+
namespace {
class SILowerControlFlow {
@@ -132,6 +137,10 @@ class SILowerControlFlow {
// Remove redundant SI_END_CF instructions.
void optimizeEndCf();
+ // Sink async DMA ops out of EXECZ-skippable then-blocks for ASYNCcnt
+ // determinism.
+ bool sinkAsyncDMAOutOfExeczBlocks(MachineFunction &MF);
+
public:
SILowerControlFlow(const GCNSubtarget *ST, LiveIntervals *LIS,
LiveVariables *LV, MachineDominatorTree *MDT,
@@ -654,6 +663,128 @@ void SILowerControlFlow::optimizeEndCf() {
}
}
+static MachineInstr *findExecRestoreAtBlockStart(MachineBasicBlock &JoinBB,
+ const SIRegisterInfo *TRI) {
+ for (MachineInstr &MI : JoinBB) {
+ if (MI.isMetaInstruction() || MI.isDebugInstr())
+ continue;
+ return MI.modifiesRegister(AMDGPU::EXEC, TRI) ? &MI : nullptr;
+ }
+ return nullptr;
+}
+
+bool SILowerControlFlow::sinkAsyncDMAOutOfExeczBlocks(MachineFunction &MF) {
+ bool Changed = false;
+
+ for (MachineBasicBlock &MBB : MF) {
+ // Header must end with an S_CBRANCH_EXECZ to JoinBB.
+ auto BrIt = llvm::find_if(MBB.terminators(), [](const MachineInstr &MI) {
+ return MI.getOpcode() == AMDGPU::S_CBRANCH_EXECZ;
+ });
+ if (BrIt == MBB.terminators().end())
+ continue;
+ MachineBasicBlock *JoinBB = BrIt->getOperand(0).getMBB();
+
+ if (MBB.succ_size() != 2)
+ continue;
+ MachineBasicBlock *S0 = *MBB.succ_begin();
+ MachineBasicBlock *S1 = *std::next(MBB.succ_begin());
+ MachineBasicBlock *ThenBB = (S0 == JoinBB) ? S1 : S0;
+ if (ThenBB == JoinBB || ThenBB->succ_size() != 1 ||
+ *ThenBB->succ_begin() != JoinBB)
+ continue;
+
+ // Sink before the EXEC restore at the top of JoinBB.
+ MachineInstr *ExecRestore = findExecRestoreAtBlockStart(*JoinBB, TRI);
+ if (!ExecRestore)
+ continue;
+
+ SmallVector<MachineInstr *, 4> ToSink;
+ bool Eligible = true;
+ for (MachineInstr &TMI : *ThenBB) {
+ if (TMI.isMetaInstruction() || TMI.isDebugInstr() || TMI.isTerminator())
+ continue;
+ if (TII->hasUnwantedEffectsWhenEXECEmpty(TMI)) {
+ Eligible = false;
+ break;
+ }
+ if (SIInstrInfo::usesASYNC_CNT(TMI)) {
+ ToSink.push_back(&TMI);
+ continue;
+ }
+ // Bail on stores that could alias the sunk DMAs.
+ if (TMI.mayStore()) {
+ Eligible = false;
+ break;
+ }
+ // Bail on non-invariant loads that could alias the sunk DMAs.
+ if (TMI.mayLoad() && !TMI.isDereferenceableInvariantLoad()) {
+ Eligible = false;
+ break;
+ }
+ }
+ if (!Eligible || ToSink.empty())
+ continue;
+
+ // Restore dominance for ThenBB-defined operands of the sunk DMAs by
+ // inserting a header-side IMPLICIT_DEF.
+ MachineBasicBlock::iterator FirstTerm = MBB.getFirstTerminator();
+ DebugLoc DL = FirstTerm->getDebugLoc();
+ SmallSet<Register, 4> NeedsImpDef;
+ for (MachineInstr *DmaMI : ToSink) {
+ for (const MachineOperand &MO : DmaMI->uses()) {
+ if (!MO.isReg() || !MO.isUse() || !MO.getReg().isVirtual())
+ continue;
+ Register Reg = MO.getReg();
+ MachineInstr *Def = MRI->getUniqueVRegDef(Reg);
+ if (!Def || Def->getParent() != ThenBB || is_contained(ToSink, Def))
+ continue;
+ if (!NeedsImpDef.insert(Reg).second)
+ continue;
+ MachineInstr *ImpDef = BuildMI(
+ MBB, FirstTerm, DL, TII->get(TargetOpcode::IMPLICIT_DEF), Reg);
+ if (LIS)
+ LIS->InsertMachineInstrInMaps(*ImpDef);
+ RecomputeRegs.insert(Reg);
+ }
+ }
+
+ // Sink each DMA before ExecRestore and propagate phys-reg uses to
+ // JoinBB live-ins.
+ for (MachineInstr *DmaMI : ToSink) {
+ LLVM_DEBUG(dbgs() << "Sinking async DMA out of execz then-block: "
+ << *DmaMI);
+ if (LIS)
+ LIS->RemoveMachineInstrFromMaps(*DmaMI);
+ JoinBB->splice(ExecRestore->getIterator(), ThenBB, DmaMI->getIterator());
+ if (LIS)
+ LIS->InsertMachineInstrInMaps(*DmaMI);
+ for (const MachineOperand &MO : DmaMI->operands()) {
+ if (!MO.isReg() || !MO.isUse())
+ continue;
+ Register Reg = MO.getReg();
+ if (Reg.isVirtual()) {
+ if (LIS)
+ RecomputeRegs.insert(Reg);
+ // Vregs defined outside ThenBB are now live through ThenBB to reach
+ // the sunk DMA in JoinBB. Update LiveVariables AliveBlocks.
+ if (LV) {
+ MachineInstr *Def = MRI->getUniqueVRegDef(Reg);
+ if (Def && Def->getParent() != ThenBB && !is_contained(ToSink, Def))
+ LV->getVarInfo(Reg).AliveBlocks.set(ThenBB->getNumber());
+ }
+ } else if (Reg != LMC.ExecReg) {
+ JoinBB->addLiveIn(Reg);
+ }
+ }
+ }
+ JoinBB->sortUniqueLiveIns();
+ Changed = true;
+ }
+
+ return Changed;
+}
+
MachineBasicBlock *SILowerControlFlow::process(MachineInstr &MI) {
MachineBasicBlock &MBB = *MI.getParent();
MachineBasicBlock::iterator I(MI);
@@ -829,6 +960,11 @@ bool SILowerControlFlow::run(MachineFunction &MF) {
}
}
+ // Sink async DMA out of s_cbranch_execz then-blocks for ASYNCcnt
+ // determinism. Must run after the main loop so the SI_END_CF anchor exists.
+ if (EnableSinkAsyncDMAOutOfExecz)
+ Changed |= sinkAsyncDMAOutOfExeczBlocks(MF);
+
optimizeEndCf();
if (LIS && Changed) {
diff --git a/llvm/test/CodeGen/AMDGPU/sink-async-dma-out-of-execz.ll b/llvm/test/CodeGen/AMDGPU/sink-async-dma-out-of-execz.ll
new file mode 100644
index 0000000000000..5f9351e63b8b2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/sink-async-dma-out-of-execz.ll
@@ -0,0 +1,216 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX1250 %s
+
+; SILowerControlFlow sinks async DMA out of s_cbranch_execz-guarded then-blocks
+; into the join-block, immediately before the s_or_b32 EXEC restore. This
+; makes ASYNCcnt deterministic regardless of whether the wave skipped the
+; block, which is required for correct waitcnt counting in software-pipelined
+; loops.
+
+
+; Basic shape: async DMA sinks into %join right before s_or_b32 exec.
+define amdgpu_ps void @async_store_simple(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr, i32 %bound) {
+; GFX1250-LABEL: async_store_simple:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: v_cmpx_lt_i32_e64 s0, v1
+; GFX1250-NEXT: ; implicit-def: $vgpr1
+; GFX1250-NEXT: ; %bb.1: ; %do_store
+; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: ; %bb.2: ; %join
+; GFX1250-NEXT: global_store_async_from_lds_b128 v1, v0, s[0:1]
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX1250-NEXT: s_wait_asynccnt 0x0
+; GFX1250-NEXT: s_endpgm
+entry:
+ %tid = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %cmp = icmp slt i32 %tid, %bound
+ br i1 %cmp, label %do_store, label %skip
+
+do_store:
+ tail call void @llvm.amdgcn.global.store.async.from.lds.b128(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr, i32 0, i32 0)
+ br label %join
+
+skip:
+ br label %join
+
+join:
+ tail call void @llvm.amdgcn.s.wait.asynccnt(i16 0)
+ ret void
+}
+
+; SWP-style pattern: address compute lives inside the masked block, so the
+; sink inserts header-side IMPLICIT_DEFs for the DMA's address vregs to
+; satisfy SSA dominance on the EXECZ-skip path.
+ at global_smem = external addrspace(3) global [0 x i8], align 16
+
+define amdgpu_kernel void @async_load_pipelined(ptr addrspace(1) readonly %src, i32 %M, i32 %N) {
+; GFX1250-LABEL: async_load_pipelined:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c nv
+; GFX1250-NEXT: v_and_b32_e32 v3, 0x7f, v0
+; GFX1250-NEXT: v_bfe_u32 v2, v0, 3, 7
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_cmp_gt_i32_e32 vcc_lo, s0, v3
+; GFX1250-NEXT: v_cmp_gt_i32_e64 s0, s1, v2
+; GFX1250-NEXT: s_and_b32 s1, vcc_lo, s0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_saveexec_b32 s0, s1
+; GFX1250-NEXT: ; implicit-def: $vgpr4
+; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT: s_cbranch_execz .LBB1_2
+; GFX1250-NEXT: ; %bb.1: ; %do_load
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 3, v3
+; GFX1250-NEXT: v_lshl_add_u32 v3, v3, 7, global_smem at abs32@lo
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 3, v2
+; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], v[4:5], v[0:1]
+; GFX1250-NEXT: v_lshl_add_u32 v4, v2, 4, v3
+; GFX1250-NEXT: .LBB1_2: ; %join
+; GFX1250-NEXT: global_load_async_to_lds_b128 v4, v[0:1], off
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_wait_asynccnt 0x0
+; GFX1250-NEXT: s_endpgm
+entry:
+ %tid = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %row = and i32 %tid, 127
+ %col = lshr i32 %tid, 3
+ %cmp_row = icmp slt i32 %row, %M
+ %cmp_col = icmp slt i32 %col, %N
+ %mask = and i1 %cmp_row, %cmp_col
+ %lds_row_off = shl nuw nsw i32 %row, 7
+ %lds_col_off = shl nuw nsw i32 %col, 4
+ %lds_base = getelementptr inbounds nuw i8, ptr addrspace(3) @global_smem, i32 %lds_row_off
+ %lds_addr = getelementptr inbounds nuw i8, ptr addrspace(3) %lds_base, i32 %lds_col_off
+ %off_row = sext i32 %row to i64
+ %src_row = getelementptr [8 x i8], ptr addrspace(1) %src, i64 %off_row
+ %off_col = sext i32 %col to i64
+ %src_addr = getelementptr [8 x i8], ptr addrspace(1) %src_row, i64 %off_col
+ br i1 %mask, label %do_load, label %skip
+
+do_load:
+ tail call void @llvm.amdgcn.global.load.async.to.lds.b128(ptr addrspace(1) %src_addr, ptr addrspace(3) %lds_addr, i32 0, i32 0)
+ br label %join
+
+skip:
+ br label %join
+
+join:
+ tail call void @llvm.amdgcn.s.wait.asynccnt(i16 0)
+ ret void
+}
+
+; Negative test: an explicit s_wait_asynccnt inside the then-block disqualifies
+; sinking. Skipping the wait on the EXECZ path would observe a different
+; ASYNCcnt value, so the eligibility scan must bail and the branch must stay.
+define amdgpu_ps void @async_load_then_block_has_waitcnt(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr, i32 %bound) {
+; GFX1250-LABEL: async_load_then_block_has_waitcnt:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: v_cmpx_lt_i32_e64 s0, v1
+; GFX1250-NEXT: s_cbranch_execz .LBB2_2
+; GFX1250-NEXT: ; %bb.1: ; %do_load
+; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: global_load_async_to_lds_b128 v0, v1, s[0:1]
+; GFX1250-NEXT: s_wait_asynccnt 0x0
+; GFX1250-NEXT: .LBB2_2: ; %join
+; GFX1250-NEXT: s_endpgm
+entry:
+ %tid = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %cmp = icmp slt i32 %tid, %bound
+ br i1 %cmp, label %do_load, label %skip
+
+do_load:
+ tail call void @llvm.amdgcn.global.load.async.to.lds.b128(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr, i32 0, i32 0)
+ tail call void @llvm.amdgcn.s.wait.asynccnt(i16 0)
+ br label %join
+
+skip:
+ br label %join
+
+join:
+ ret void
+}
+
+; Negative test: a non-invariant DS load reading from the LDS region the
+; async load writes to would read stale data if the DMA were sunk past it.
+; The mayLoad bail must fire.
+define amdgpu_ps void @async_load_then_block_has_aliasing_lds_load(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr, ptr addrspace(1) %out, i32 %bound) {
+; GFX1250-LABEL: async_load_then_block_has_aliasing_lds_load:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
+; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: v_cmpx_lt_i32_e64 s0, v3
+; GFX1250-NEXT: s_cbranch_execz .LBB3_2
+; GFX1250-NEXT: ; %bb.1: ; %do_load
+; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: global_load_async_to_lds_b128 v0, v1, s[0:1]
+; GFX1250-NEXT: ds_load_b32 v0, v0
+; GFX1250-NEXT: s_wait_dscnt 0x0
+; GFX1250-NEXT: global_store_b32 v[4:5], v0, off
+; GFX1250-NEXT: .LBB3_2: ; %join
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX1250-NEXT: s_wait_asynccnt 0x0
+; GFX1250-NEXT: s_endpgm
+entry:
+ %tid = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %cmp = icmp slt i32 %tid, %bound
+ br i1 %cmp, label %do_load, label %skip
+
+do_load:
+ tail call void @llvm.amdgcn.global.load.async.to.lds.b128(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr, i32 0, i32 0)
+ %val = load i32, ptr addrspace(3) %laddr
+ store i32 %val, ptr addrspace(1) %out
+ br label %join
+
+skip:
+ br label %join
+
+join:
+ tail call void @llvm.amdgcn.s.wait.asynccnt(i16 0)
+ ret void
+}
+
+; Negative test: a regular store to the same global address the async load
+define amdgpu_ps void @async_load_then_block_has_aliasing_global_store(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr, i32 %bound) {
+; GFX1250-LABEL: async_load_then_block_has_aliasing_global_store:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_cmp_lt_i32_e32 vcc_lo, s0, v3
+; GFX1250-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-NEXT: s_cbranch_execz .LBB4_2
+; GFX1250-NEXT: ; %bb.1: ; %do_load
+; GFX1250-NEXT: global_load_async_to_lds_b128 v2, v[0:1], off
+; GFX1250-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
+; GFX1250-NEXT: .LBB4_2: ; %join
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_wait_asynccnt 0x0
+; GFX1250-NEXT: s_endpgm
+entry:
+ %tid = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %cmp = icmp slt i32 %tid, %bound
+ br i1 %cmp, label %do_load, label %skip
+
+do_load:
+ tail call void @llvm.amdgcn.global.load.async.to.lds.b128(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr, i32 0, i32 0)
+ store i32 0, ptr addrspace(1) %gaddr
+ br label %join
+
+skip:
+ br label %join
+
+join:
+ tail call void @llvm.amdgcn.s.wait.asynccnt(i16 0)
+ ret void
+}
>From 72eae27c7fea6e61a4d9ee781dd0470011ef92b3 Mon Sep 17 00:00:00 2001
From: Vigneshwar <vjayakum at amd.com>
Date: Thu, 23 Jul 2026 14:44:31 -0500
Subject: [PATCH 2/3] moved it to a new pass.
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 3 +
llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 1 +
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 6 +
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 +
llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp | 136 ------
llvm/lib/Target/AMDGPU/SISinkAsyncDMA.cpp | 253 ++++++++++
llvm/lib/Target/AMDGPU/SISinkAsyncDMA.h | 22 +
llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll | 3 +
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 5 +
.../test/CodeGen/AMDGPU/si-sink-async-dma.mir | 441 ++++++++++++++++++
.../AMDGPU/sink-async-dma-out-of-execz.ll | 159 +------
11 files changed, 744 insertions(+), 286 deletions(-)
create mode 100644 llvm/lib/Target/AMDGPU/SISinkAsyncDMA.cpp
create mode 100644 llvm/lib/Target/AMDGPU/SISinkAsyncDMA.h
create mode 100644 llvm/test/CodeGen/AMDGPU/si-sink-async-dma.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index c6dd1dbb62449..a4d96e2c06852 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -235,6 +235,9 @@ extern char &SIWholeQuadModeID;
void initializeSILowerControlFlowLegacyPass(PassRegistry &);
extern char &SILowerControlFlowLegacyID;
+void initializeSISinkAsyncDMALegacyPass(PassRegistry &);
+extern char &SISinkAsyncDMALegacyID;
+
void initializeSIPreEmitPeepholeLegacyPass(PassRegistry &);
extern char &SIPreEmitPeepholeID;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 2a6560b309e62..fff6d4ee20860 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -152,6 +152,7 @@ MACHINE_FUNCTION_PASS("si-post-ra-bundler", SIPostRABundlerPass())
MACHINE_FUNCTION_PASS("si-pre-allocate-wwm-regs", SIPreAllocateWWMRegsPass())
MACHINE_FUNCTION_PASS("si-pre-emit-peephole", SIPreEmitPeepholePass())
MACHINE_FUNCTION_PASS("si-shrink-instructions", SIShrinkInstructionsPass())
+MACHINE_FUNCTION_PASS("si-sink-async-dma", SISinkAsyncDMAPass())
MACHINE_FUNCTION_PASS("si-wqm", SIWholeQuadModePass())
#undef MACHINE_FUNCTION_PASS
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 65317016c6390..dd26510294eeb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -66,6 +66,7 @@
#include "SIPostRABundler.h"
#include "SIPreAllocateWWMRegs.h"
#include "SIShrinkInstructions.h"
+#include "SISinkAsyncDMA.h"
#include "SIWholeQuadMode.h"
#include "TargetInfo/AMDGPUTargetInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
@@ -713,6 +714,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
initializeSIModeRegisterLegacyPass(*PR);
initializeSIWholeQuadModeLegacyPass(*PR);
initializeSILowerControlFlowLegacyPass(*PR);
+ initializeSISinkAsyncDMALegacyPass(*PR);
initializeSIPreEmitPeepholeLegacyPass(*PR);
initializeSILateBranchLoweringLegacyPass(*PR);
initializeSIMemoryLegalizerLegacyPass(*PR);
@@ -1702,6 +1704,7 @@ void GCNPassConfig::addFastRegAlloc() {
// TwoAddressInstructions, otherwise the processing of the tied operand of
// SI_ELSE will introduce a copy of the tied operand source after the else.
insertPass(&PHIEliminationID, &SILowerControlFlowLegacyID);
+ insertPass(&SILowerControlFlowLegacyID, &SISinkAsyncDMALegacyID);
insertPass(&TwoAddressInstructionPassID, &SIWholeQuadModeID);
@@ -1728,6 +1731,7 @@ void GCNPassConfig::addOptimizedRegAlloc() {
// TwoAddressInstructions, otherwise the processing of the tied operand of
// SI_ELSE will introduce a copy of the tied operand source after the else.
insertPass(&PHIEliminationID, &SILowerControlFlowLegacyID);
+ insertPass(&SILowerControlFlowLegacyID, &SISinkAsyncDMALegacyID);
if (EnableRewritePartialRegUses)
insertPass(&RenameIndependentSubregsID, &GCNRewritePartialRegUsesID);
@@ -2438,6 +2442,7 @@ void AMDGPUCodeGenPassBuilder::addMachineSSAOptimization(
Error AMDGPUCodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) const {
insertPass<PHIEliminationPass>(SILowerControlFlowPass());
+ insertPass<PHIEliminationPass>(SISinkAsyncDMAPass());
insertPass<TwoAddressInstructionPass>(SIWholeQuadModePass());
@@ -2500,6 +2505,7 @@ Error AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(
// TwoAddressInstructions, otherwise the processing of the tied operand of
// SI_ELSE will introduce a copy of the tied operand source after the else.
insertPass<PHIEliminationPass>(SILowerControlFlowPass());
+ insertPass<PHIEliminationPass>(SISinkAsyncDMAPass());
if (EnableRewritePartialRegUses)
insertPass<RenameIndependentSubregsPass>(GCNRewritePartialRegUsesPass());
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index ae8f1c0fad5ba..3c0537a1980ad 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -177,6 +177,7 @@ add_llvm_target(AMDGPUCodeGen
SISpillUtils.cpp
SIMachineScheduler.cpp
SIMemoryLegalizer.cpp
+ SISinkAsyncDMA.cpp
SIModeRegister.cpp
SIModeRegisterDefaults.cpp
SIOptimizeExecMasking.cpp
diff --git a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
index 1efc15233f080..9cc86e84407b1 100644
--- a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
@@ -70,11 +70,6 @@ static cl::opt<bool>
RemoveRedundantEndcf("amdgpu-remove-redundant-endcf",
cl::init(true), cl::ReallyHidden);
-static cl::opt<bool> EnableSinkAsyncDMAOutOfExecz(
- "amdgpu-sink-async-dma-out-of-execz", cl::init(true), cl::ReallyHidden,
- cl::desc("Sink async DMA out of EXECZ-skipped then-blocks for "
- "ASYNCcnt determinism"));
-
namespace {
class SILowerControlFlow {
@@ -137,10 +132,6 @@ class SILowerControlFlow {
// Remove redundant SI_END_CF instructions.
void optimizeEndCf();
- // Sink async DMA ops out of EXECZ-skippable then-blocks for ASYNCcnt
- // determinism.
- bool sinkAsyncDMAOutOfExeczBlocks(MachineFunction &MF);
-
public:
SILowerControlFlow(const GCNSubtarget *ST, LiveIntervals *LIS,
LiveVariables *LV, MachineDominatorTree *MDT,
@@ -663,128 +654,6 @@ void SILowerControlFlow::optimizeEndCf() {
}
}
-static MachineInstr *findExecRestoreAtBlockStart(MachineBasicBlock &JoinBB,
- const SIRegisterInfo *TRI) {
- for (MachineInstr &MI : JoinBB) {
- if (MI.isMetaInstruction() || MI.isDebugInstr())
- continue;
- return MI.modifiesRegister(AMDGPU::EXEC, TRI) ? &MI : nullptr;
- }
- return nullptr;
-}
-
-bool SILowerControlFlow::sinkAsyncDMAOutOfExeczBlocks(MachineFunction &MF) {
- bool Changed = false;
-
- for (MachineBasicBlock &MBB : MF) {
- // Header must end with an S_CBRANCH_EXECZ to JoinBB.
- auto BrIt = llvm::find_if(MBB.terminators(), [](const MachineInstr &MI) {
- return MI.getOpcode() == AMDGPU::S_CBRANCH_EXECZ;
- });
- if (BrIt == MBB.terminators().end())
- continue;
- MachineBasicBlock *JoinBB = BrIt->getOperand(0).getMBB();
-
- if (MBB.succ_size() != 2)
- continue;
- MachineBasicBlock *S0 = *MBB.succ_begin();
- MachineBasicBlock *S1 = *std::next(MBB.succ_begin());
- MachineBasicBlock *ThenBB = (S0 == JoinBB) ? S1 : S0;
- if (ThenBB == JoinBB || ThenBB->succ_size() != 1 ||
- *ThenBB->succ_begin() != JoinBB)
- continue;
-
- // Sink before the EXEC restore at the top of JoinBB.
- MachineInstr *ExecRestore = findExecRestoreAtBlockStart(*JoinBB, TRI);
- if (!ExecRestore)
- continue;
-
- SmallVector<MachineInstr *, 4> ToSink;
- bool Eligible = true;
- for (MachineInstr &TMI : *ThenBB) {
- if (TMI.isMetaInstruction() || TMI.isDebugInstr() || TMI.isTerminator())
- continue;
- if (TII->hasUnwantedEffectsWhenEXECEmpty(TMI)) {
- Eligible = false;
- break;
- }
- if (SIInstrInfo::usesASYNC_CNT(TMI)) {
- ToSink.push_back(&TMI);
- continue;
- }
- // Bail on stores that could alias the sunk DMAs.
- if (TMI.mayStore()) {
- Eligible = false;
- break;
- }
- // Bail on non-invariant loads that could alias the sunk DMAs.
- if (TMI.mayLoad() && !TMI.isDereferenceableInvariantLoad()) {
- Eligible = false;
- break;
- }
- }
- if (!Eligible || ToSink.empty())
- continue;
-
- // Restore dominance for ThenBB-defined operands of the sunk DMAs by
- // inserting a header-side IMPLICIT_DEF.
- MachineBasicBlock::iterator FirstTerm = MBB.getFirstTerminator();
- DebugLoc DL = FirstTerm->getDebugLoc();
- SmallSet<Register, 4> NeedsImpDef;
- for (MachineInstr *DmaMI : ToSink) {
- for (const MachineOperand &MO : DmaMI->uses()) {
- if (!MO.isReg() || !MO.isUse() || !MO.getReg().isVirtual())
- continue;
- Register Reg = MO.getReg();
- MachineInstr *Def = MRI->getUniqueVRegDef(Reg);
- if (!Def || Def->getParent() != ThenBB || is_contained(ToSink, Def))
- continue;
- if (!NeedsImpDef.insert(Reg).second)
- continue;
- MachineInstr *ImpDef = BuildMI(
- MBB, FirstTerm, DL, TII->get(TargetOpcode::IMPLICIT_DEF), Reg);
- if (LIS)
- LIS->InsertMachineInstrInMaps(*ImpDef);
- RecomputeRegs.insert(Reg);
- }
- }
-
- // Sink each DMA before ExecRestore and propagate phys-reg uses to
- // JoinBB live-ins.
- for (MachineInstr *DmaMI : ToSink) {
- LLVM_DEBUG(dbgs() << "Sinking async DMA out of execz then-block: "
- << *DmaMI);
- if (LIS)
- LIS->RemoveMachineInstrFromMaps(*DmaMI);
- JoinBB->splice(ExecRestore->getIterator(), ThenBB, DmaMI->getIterator());
- if (LIS)
- LIS->InsertMachineInstrInMaps(*DmaMI);
- for (const MachineOperand &MO : DmaMI->operands()) {
- if (!MO.isReg() || !MO.isUse())
- continue;
- Register Reg = MO.getReg();
- if (Reg.isVirtual()) {
- if (LIS)
- RecomputeRegs.insert(Reg);
- // Vregs defined outside ThenBB are now live through ThenBB to reach
- // the sunk DMA in JoinBB. Update LiveVariables AliveBlocks.
- if (LV) {
- MachineInstr *Def = MRI->getUniqueVRegDef(Reg);
- if (Def && Def->getParent() != ThenBB && !is_contained(ToSink, Def))
- LV->getVarInfo(Reg).AliveBlocks.set(ThenBB->getNumber());
- }
- } else if (Reg != LMC.ExecReg) {
- JoinBB->addLiveIn(Reg);
- }
- }
- }
- JoinBB->sortUniqueLiveIns();
- Changed = true;
- }
-
- return Changed;
-}
-
MachineBasicBlock *SILowerControlFlow::process(MachineInstr &MI) {
MachineBasicBlock &MBB = *MI.getParent();
MachineBasicBlock::iterator I(MI);
@@ -960,11 +829,6 @@ bool SILowerControlFlow::run(MachineFunction &MF) {
}
}
- // Sink async DMA out of s_cbranch_execz then-blocks for ASYNCcnt
- // determinism. Must run after the main loop so the SI_END_CF anchor exists.
- if (EnableSinkAsyncDMAOutOfExecz)
- Changed |= sinkAsyncDMAOutOfExeczBlocks(MF);
-
optimizeEndCf();
if (LIS && Changed) {
diff --git a/llvm/lib/Target/AMDGPU/SISinkAsyncDMA.cpp b/llvm/lib/Target/AMDGPU/SISinkAsyncDMA.cpp
new file mode 100644
index 0000000000000..07c28c8717357
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/SISinkAsyncDMA.cpp
@@ -0,0 +1,253 @@
+//===-- SISinkAsyncDMA.cpp - Sink async DMA out of execz then-blocks ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// LLVM lowers a divergent branch around global_load_async_to_lds /
+/// global_store_async_from_lds with an S_CBRANCH_EXECZ. Fully-masked waves
+/// therefore skip the DMA entirely, which makes the ASYNCcnt observed at the
+/// join point depend on whether the wave took the branch. Software-pipelined
+/// kernels must then use conservative async waitcnts.
+///
+/// This pass sinks each async DMA out of such a then-block into the join
+/// block, immediately before the EXEC-mask restore (the S_OR that ends the
+/// control-flow region). EXEC at the sunk slot is the same masked value that
+/// guarded the then-block, so per-lane behavior is unchanged, but every wave
+/// now issues the DMA and ASYNCcnt becomes deterministic.
+///
+/// This runs right after SILowerControlFlow so the EXEC restore anchor exists,
+/// and before waitcnt insertion so the improved counts can be used.
+///
+/// This is determinism-only: it never relaxes or rewrites a wait (an
+/// s_wait_asynccnt 0 stays 0). Correctness never depends on the transform
+/// firing; with the pass disabled the code is still correct, just conservative.
+//
+//===----------------------------------------------------------------------===//
+
+#include "SISinkAsyncDMA.h"
+#include "AMDGPU.h"
+#include "AMDGPULaneMaskUtils.h"
+#include "GCNSubtarget.h"
+#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "SIInstrInfo.h"
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/CodeGen/LiveVariables.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "si-sink-async-dma"
+
+namespace {
+
+class SISinkAsyncDMA {
+ const SIInstrInfo *TII = nullptr;
+ const SIRegisterInfo *TRI = nullptr;
+ MachineRegisterInfo *MRI = nullptr;
+ LiveVariables *LV = nullptr;
+ const AMDGPU::LaneMaskConstants &LMC;
+
+ bool sinkFromBlock(MachineBasicBlock &MBB);
+
+public:
+ SISinkAsyncDMA(const GCNSubtarget *ST, LiveVariables *LV)
+ : TII(ST->getInstrInfo()), TRI(&TII->getRegisterInfo()), LV(LV),
+ LMC(AMDGPU::LaneMaskConstants::get(*ST)) {}
+
+ bool run(MachineFunction &MF);
+};
+
+class SISinkAsyncDMALegacy : public MachineFunctionPass {
+public:
+ static char ID;
+
+ SISinkAsyncDMALegacy() : MachineFunctionPass(ID) {}
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+
+ StringRef getPassName() const override {
+ return "SI sink async DMA out of execz then-blocks";
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ AU.addUsedIfAvailable<LiveVariablesWrapperPass>();
+ AU.addPreserved<LiveVariablesWrapperPass>();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+};
+
+} // namespace
+
+char SISinkAsyncDMALegacy::ID = 0;
+
+INITIALIZE_PASS(SISinkAsyncDMALegacy, DEBUG_TYPE,
+ "SI sink async DMA out of execz then-blocks", false, false)
+
+char &llvm::SISinkAsyncDMALegacyID = SISinkAsyncDMALegacy::ID;
+
+/// Return the S_OR EXEC restore at the top of \p JoinBB, if present.
+static MachineInstr *findExecRestore(MachineBasicBlock &JoinBB,
+ const AMDGPU::LaneMaskConstants &LMC) {
+ auto I = JoinBB.getFirstNonDebugInstr();
+ return I != JoinBB.end() && I->getOpcode() == LMC.OrOpc &&
+ I->getOperand(0).getReg() == LMC.ExecReg &&
+ I->getOperand(1).getReg() == LMC.ExecReg
+ ? &*I
+ : nullptr;
+}
+
+static bool isAsyncDMA(const MachineInstr &MI) {
+ return SIInstrInfo::isLDSDMA(MI) && SIInstrInfo::usesASYNC_CNT(MI);
+}
+
+bool SISinkAsyncDMA::sinkFromBlock(MachineBasicBlock &MBB) {
+ if (MBB.succ_size() != 2)
+ return false;
+
+ MachineBasicBlock *TBB = nullptr;
+ MachineBasicBlock *FBB = nullptr;
+ SmallVector<MachineOperand, 4> Cond;
+ if (TII->analyzeBranch(MBB, TBB, FBB, Cond) || !TBB || Cond.empty())
+ return false;
+
+ auto CondBr = find_if(MBB.terminators(), [](const MachineInstr &MI) {
+ return MI.isConditionalBranch();
+ });
+ if (CondBr == MBB.terminators().end() ||
+ CondBr->getOpcode() != AMDGPU::S_CBRANCH_EXECZ)
+ return false;
+
+ MachineBasicBlock *JoinBB = TBB;
+ MachineBasicBlock *S0 = *MBB.succ_begin();
+ MachineBasicBlock *S1 = *std::next(MBB.succ_begin());
+ MachineBasicBlock *ThenBB = (S0 == JoinBB) ? S1 : S0;
+ if (ThenBB == JoinBB || ThenBB->succ_size() != 1 ||
+ *ThenBB->succ_begin() != JoinBB)
+ return false;
+
+ // A third incoming edge could reach the DMA under an unrelated EXEC.
+ if (JoinBB->pred_size() != 2)
+ return false;
+
+ MachineInstr *ExecRestore = findExecRestore(*JoinBB, LMC);
+ if (!ExecRestore)
+ return false;
+
+ SmallVector<MachineInstr *, 4> ToSink;
+ for (MachineInstr &TMI : *ThenBB) {
+ if (TMI.isMetaInstruction() || TMI.isTerminator())
+ continue;
+ if (TII->hasUnwantedEffectsWhenEXECEmpty(TMI))
+ return false;
+ if (isAsyncDMA(TMI)) {
+ ToSink.push_back(&TMI);
+ continue;
+ }
+ if (TMI.modifiesRegister(AMDGPU::EXEC, TRI) ||
+ TMI.hasUnmodeledSideEffects())
+ return false;
+
+ if (ToSink.empty())
+ continue;
+ if (TMI.mayStore())
+ return false;
+ if (TMI.mayLoad() && !TMI.isDereferenceableInvariantLoad())
+ return false;
+
+ // Reject dependencies with earlier DMAs that would cross this instruction.
+ for (const MachineOperand &MO : TMI.operands()) {
+ if (!MO.isReg() || !MO.getReg() || (!MO.isDef() && !MO.readsReg()) ||
+ TRI->regsOverlap(MO.getReg(), AMDGPU::EXEC))
+ continue;
+ if (any_of(ToSink, [&](const MachineInstr *DmaMI) {
+ return DmaMI->readsRegister(MO.getReg(), TRI) ||
+ DmaMI->modifiesRegister(MO.getReg(), TRI);
+ }))
+ return false;
+ }
+ }
+ if (ToSink.empty())
+ return false;
+
+ SmallSet<Register, 4> NeedsImpDef;
+ SmallSet<Register, 8> LiveThroughThen;
+ for (const MachineInstr *DmaMI : ToSink) {
+ for (const MachineOperand &MO : DmaMI->uses()) {
+ if (!MO.isReg() || !MO.readsReg() || !MO.getReg().isVirtual())
+ continue;
+ Register Reg = MO.getReg();
+ MachineInstr *Def = MRI->getUniqueVRegDef(Reg);
+ if (!Def)
+ return false;
+ if (Def->getParent() == ThenBB) {
+ if (!isAsyncDMA(*Def))
+ NeedsImpDef.insert(Reg);
+ } else {
+ LiveThroughThen.insert(Reg);
+ }
+ }
+ }
+
+ auto FirstTerm = MBB.getFirstTerminator();
+ for (Register Reg : NeedsImpDef)
+ BuildMI(MBB, FirstTerm, FirstTerm->getDebugLoc(),
+ TII->get(TargetOpcode::IMPLICIT_DEF), Reg);
+
+ for (MachineInstr *DmaMI : ToSink) {
+ LLVM_DEBUG(dbgs() << "Sinking async DMA out of execz then-block: "
+ << *DmaMI);
+ DmaMI->moveBefore(ExecRestore);
+ for (const MachineOperand &MO : DmaMI->uses()) {
+ if (!MO.isReg() || !MO.readsReg())
+ continue;
+ Register Reg = MO.getReg();
+ if (Reg.isPhysical() && Reg != LMC.ExecReg)
+ JoinBB->addLiveIn(Reg);
+ }
+ }
+
+ if (LV)
+ for (Register Reg : LiveThroughThen)
+ LV->getVarInfo(Reg).AliveBlocks.set(ThenBB->getNumber());
+
+ JoinBB->sortUniqueLiveIns();
+ return true;
+}
+
+bool SISinkAsyncDMA::run(MachineFunction &MF) {
+ MRI = &MF.getRegInfo();
+
+ bool Changed = false;
+ for (MachineBasicBlock &MBB : MF)
+ Changed |= sinkFromBlock(MBB);
+
+ return Changed;
+}
+
+bool SISinkAsyncDMALegacy::runOnMachineFunction(MachineFunction &MF) {
+ const GCNSubtarget *ST = &MF.getSubtarget<GCNSubtarget>();
+ auto *LVWrapper = getAnalysisIfAvailable<LiveVariablesWrapperPass>();
+ LiveVariables *LV = LVWrapper ? &LVWrapper->getLV() : nullptr;
+ return SISinkAsyncDMA(ST, LV).run(MF);
+}
+
+PreservedAnalyses
+SISinkAsyncDMAPass::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM) {
+ const GCNSubtarget *ST = &MF.getSubtarget<GCNSubtarget>();
+ LiveVariables *LV = MFAM.getCachedResult<LiveVariablesAnalysis>(MF);
+
+ bool Changed = SISinkAsyncDMA(ST, LV).run(MF);
+ if (!Changed)
+ return PreservedAnalyses::all();
+
+ auto PA = getMachineFunctionPassPreservedAnalyses();
+ PA.preserveSet<CFGAnalyses>();
+ PA.preserve<LiveVariablesAnalysis>();
+ return PA;
+}
diff --git a/llvm/lib/Target/AMDGPU/SISinkAsyncDMA.h b/llvm/lib/Target/AMDGPU/SISinkAsyncDMA.h
new file mode 100644
index 0000000000000..93189b6a69399
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/SISinkAsyncDMA.h
@@ -0,0 +1,22 @@
+//===- SISinkAsyncDMA.h -----------------------------------------*- C++- *-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_SISINKASYNCDMA_H
+#define LLVM_LIB_TARGET_AMDGPU_SISINKASYNCDMA_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+class SISinkAsyncDMAPass : public PassInfoMixin<SISinkAsyncDMAPass> {
+public:
+ PreservedAnalyses run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM);
+};
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_SISINKASYNCDMA_H
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
index c49b2b927bd31..a5cd9f904eef1 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
@@ -59,6 +59,7 @@
; GCN-O0-NEXT: cgscc(function(machine-function(reg-usage-propagation
; GCN-O0-NEXT: phi-node-elimination
; GCN-O0-NEXT: si-lower-control-flow
+; GCN-O0-NEXT: si-sink-async-dma
; GCN-O0-NEXT: two-address-instruction
; GCN-O0-NEXT: si-wqm
; GCN-O0-NEXT: amdgpu-pre-ra-long-branch-reg
@@ -197,6 +198,7 @@
; GCN-O2-NEXT: require<machine-loops>
; GCN-O2-NEXT: phi-node-elimination
; GCN-O2-NEXT: si-lower-control-flow
+; GCN-O2-NEXT: si-sink-async-dma
; GCN-O2-NEXT: two-address-instruction
; GCN-O2-NEXT: register-coalescer
; GCN-O2-NEXT: rename-independent-subregs
@@ -366,6 +368,7 @@
; GCN-O3-NEXT: require<machine-loops>
; GCN-O3-NEXT: phi-node-elimination
; GCN-O3-NEXT: si-lower-control-flow
+; GCN-O3-NEXT: si-sink-async-dma
; GCN-O3-NEXT: two-address-instruction
; GCN-O3-NEXT: register-coalescer
; GCN-O3-NEXT: rename-independent-subregs
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index 070c873798647..41a44a0b1b4a3 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -111,6 +111,7 @@
; GCN-O0-NEXT: Register Usage Information Propagation
; GCN-O0-NEXT: Eliminate PHI nodes for register allocation
; GCN-O0-NEXT: SI Lower control flow pseudo instructions
+; GCN-O0-NEXT: SI sink async DMA out of execz then-blocks
; GCN-O0-NEXT: Two-Address instruction pass
; GCN-O0-NEXT: MachineDominator Tree Construction
; GCN-O0-NEXT: Slot index numbering
@@ -354,6 +355,7 @@
; GCN-O1-NEXT: SI Optimize VGPR LiveRange
; GCN-O1-NEXT: Eliminate PHI nodes for register allocation
; GCN-O1-NEXT: SI Lower control flow pseudo instructions
+; GCN-O1-NEXT: SI sink async DMA out of execz then-blocks
; GCN-O1-NEXT: Two-Address instruction pass
; GCN-O1-NEXT: Slot index numbering
; GCN-O1-NEXT: Live Interval Analysis
@@ -671,6 +673,7 @@
; GCN-O1-OPTS-NEXT: SI Optimize VGPR LiveRange
; GCN-O1-OPTS-NEXT: Eliminate PHI nodes for register allocation
; GCN-O1-OPTS-NEXT: SI Lower control flow pseudo instructions
+; GCN-O1-OPTS-NEXT: SI sink async DMA out of execz then-blocks
; GCN-O1-OPTS-NEXT: Two-Address instruction pass
; GCN-O1-OPTS-NEXT: Slot index numbering
; GCN-O1-OPTS-NEXT: Live Interval Analysis
@@ -993,6 +996,7 @@
; GCN-O2-NEXT: SI Optimize VGPR LiveRange
; GCN-O2-NEXT: Eliminate PHI nodes for register allocation
; GCN-O2-NEXT: SI Lower control flow pseudo instructions
+; GCN-O2-NEXT: SI sink async DMA out of execz then-blocks
; GCN-O2-NEXT: Two-Address instruction pass
; GCN-O2-NEXT: Slot index numbering
; GCN-O2-NEXT: Live Interval Analysis
@@ -1329,6 +1333,7 @@
; GCN-O3-NEXT: SI Optimize VGPR LiveRange
; GCN-O3-NEXT: Eliminate PHI nodes for register allocation
; GCN-O3-NEXT: SI Lower control flow pseudo instructions
+; GCN-O3-NEXT: SI sink async DMA out of execz then-blocks
; GCN-O3-NEXT: Two-Address instruction pass
; GCN-O3-NEXT: Slot index numbering
; GCN-O3-NEXT: Live Interval Analysis
diff --git a/llvm/test/CodeGen/AMDGPU/si-sink-async-dma.mir b/llvm/test/CodeGen/AMDGPU/si-sink-async-dma.mir
new file mode 100644
index 0000000000000..77674b7e7b415
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/si-sink-async-dma.mir
@@ -0,0 +1,441 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=si-sink-async-dma -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -passes=si-sink-async-dma -verify-machineinstrs -o - %s | FileCheck %s
+
+# Sink an async store immediately before the EXEC restore.
+---
+name: sink_async_store
+tracksRegLiveness: true
+isSSA: false
+machineFunctionInfo:
+ isEntryFunction: true
+ sgprForEXECCopy: '$sgpr105'
+body: |
+ ; CHECK-LABEL: name: sink_async_store
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: liveins: $asynccnt, $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR killed [[DEF1]], killed [[DEF2]], killed [[DEF]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: $exec_lo = S_OR_B32 $exec_lo, killed [[COPY]], implicit-def $scc
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1, %bb.2
+ %3:vgpr_32 = IMPLICIT_DEF
+ %0:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
+ S_CBRANCH_EXECZ %bb.2, implicit $exec
+ S_BRANCH %bb.1
+
+ bb.1:
+ %6:sgpr_64 = IMPLICIT_DEF
+ %9:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR killed %6, killed %9, killed %3, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+
+ bb.2:
+ $exec_lo = S_OR_B32 $exec_lo, killed %0, implicit-def $scc
+ S_ENDPGM 0
+...
+
+# Any non-DMA store prevents sinking because the pass has no alias information.
+---
+name: no_sink_other_store
+tracksRegLiveness: true
+isSSA: false
+machineFunctionInfo:
+ isEntryFunction: true
+ sgprForEXECCopy: '$sgpr105'
+body: |
+ ; CHECK-LABEL: name: no_sink_other_store
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
+ ; CHECK-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed [[DEF]], [[DEF1]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD killed [[DEF1]], killed [[V_MOV_B32_e32_]], 0, 0, implicit $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: $exec_lo = S_OR_B32 $exec_lo, killed [[COPY]], implicit-def $scc
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1, %bb.2
+ %3:vgpr_32 = IMPLICIT_DEF
+ %12:vreg_64_align2 = IMPLICIT_DEF
+ %0:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
+ S_CBRANCH_EXECZ %bb.2, implicit $exec
+ S_BRANCH %bb.1
+
+ bb.1:
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed %3, %12, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ %9:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ GLOBAL_STORE_DWORD killed %12, killed %9, 0, 0, implicit $exec
+
+ bb.2:
+ $exec_lo = S_OR_B32 $exec_lo, killed %0, implicit-def $scc
+ S_ENDPGM 0
+...
+
+# Sink an async load and repair its block-local address definition.
+---
+name: sink_async_load
+tracksRegLiveness: true
+isSSA: false
+machineFunctionInfo:
+ isEntryFunction: true
+ sgprForEXECCopy: '$sgpr105'
+body: |
+ ; CHECK-LABEL: name: sink_async_load
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: liveins: $asynccnt, $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed [[DEF]], killed [[DEF1]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: $exec_lo = S_OR_B32 $exec_lo, killed [[COPY]], implicit-def $scc
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1, %bb.2
+ %3:vgpr_32 = IMPLICIT_DEF
+ %0:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
+ S_CBRANCH_EXECZ %bb.2, implicit $exec
+ S_BRANCH %bb.1
+
+ bb.1:
+ %12:vreg_64_align2 = IMPLICIT_DEF
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed %3, killed %12, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+
+ bb.2:
+ $exec_lo = S_OR_B32 $exec_lo, killed %0, implicit-def $scc
+ S_ENDPGM 0
+...
+
+# An ASYNC_CNT barrier is not an LDS DMA candidate.
+---
+name: no_sink_async_barrier_arrive
+tracksRegLiveness: true
+isSSA: false
+machineFunctionInfo:
+ isEntryFunction: true
+ sgprForEXECCopy: '$sgpr105'
+body: |
+ ; CHECK-LABEL: name: no_sink_async_barrier_arrive
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
+ ; CHECK-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 killed [[DEF]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: $exec_lo = S_OR_B32 $exec_lo, killed [[COPY]], implicit-def $scc
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1, %bb.2
+ %2:vgpr_32 = IMPLICIT_DEF
+ %0:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
+ S_CBRANCH_EXECZ %bb.2, implicit $exec
+ S_BRANCH %bb.1
+
+ bb.1:
+ DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 killed %2, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+
+ bb.2:
+ $exec_lo = S_OR_B32 $exec_lo, killed %0, implicit-def $scc
+ S_ENDPGM 0
+...
+
+# %12 has multiple defs, so the pass cannot add a header IMPLICIT_DEF that makes
+# its sunk use well-formed on the skip path.
+---
+name: no_sink_multiple_defs
+tracksRegLiveness: true
+isSSA: false
+machineFunctionInfo:
+ isEntryFunction: true
+ sgprForEXECCopy: '$sgpr105'
+body: |
+ ; CHECK-LABEL: name: no_sink_multiple_defs
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
+ ; CHECK-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed [[DEF]], killed [[DEF1]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: $exec_lo = S_OR_B32 $exec_lo, killed [[COPY]], implicit-def $scc
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1, %bb.2
+ %3:vgpr_32 = IMPLICIT_DEF
+ %0:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
+ S_CBRANCH_EXECZ %bb.2, implicit $exec
+ S_BRANCH %bb.1
+
+ bb.1:
+ %12:vreg_64_align2 = IMPLICIT_DEF
+ %12:vreg_64_align2 = IMPLICIT_DEF
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed %3, killed %12, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+
+ bb.2:
+ $exec_lo = S_OR_B32 $exec_lo, killed %0, implicit-def $scc
+ S_ENDPGM 0
+...
+
+# A post-DMA dependency on an operand prevents sinking.
+---
+name: no_sink_post_dma_dependency
+tracksRegLiveness: true
+isSSA: false
+machineFunctionInfo:
+ isEntryFunction: true
+ sgprForEXECCopy: '$sgpr105'
+body: |
+ ; CHECK-LABEL: name: no_sink_post_dma_dependency
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
+ ; CHECK-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 [[DEF]], killed [[DEF1]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY killed [[DEF]]
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: $exec_lo = S_OR_B32 $exec_lo, killed [[COPY]], implicit-def $scc
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1, %bb.2
+ %3:vgpr_32 = IMPLICIT_DEF
+ %12:vreg_64_align2 = IMPLICIT_DEF
+ %0:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
+ S_CBRANCH_EXECZ %bb.2, implicit $exec
+ S_BRANCH %bb.1
+
+ bb.1:
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128 %3, killed %12, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ %20:vgpr_32 = COPY killed %3
+
+ bb.2:
+ $exec_lo = S_OR_B32 $exec_lo, killed %0, implicit-def $scc
+ S_ENDPGM 0
+...
+
+# A third join predecessor is outside the EXECZ region, so the DMA must stay in
+# bb.2.
+---
+name: no_sink_extra_join_predecessor
+tracksRegLiveness: true
+isSSA: false
+machineFunctionInfo:
+ isEntryFunction: true
+ sgprForEXECCopy: '$sgpr105'
+body: |
+ ; CHECK-LABEL: name: no_sink_extra_join_predecessor
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.3, implicit undef $scc
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.4(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_CBRANCH_EXECZ %bb.4, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.4(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR [[DEF1]], [[DEF2]], [[DEF]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: S_BRANCH %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_BRANCH %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: $exec_lo = S_OR_B32 $exec_lo, [[COPY]], implicit-def $scc
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1, %bb.3
+ %3:vgpr_32 = IMPLICIT_DEF
+ %6:sgpr_64 = IMPLICIT_DEF
+ %9:vgpr_32 = IMPLICIT_DEF
+ %0:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
+ S_CBRANCH_SCC1 %bb.3, implicit undef $scc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2, %bb.4
+ S_CBRANCH_EXECZ %bb.4, implicit $exec
+ S_BRANCH %bb.2
+
+ bb.2:
+ successors: %bb.4
+ GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR %6, %9, %3, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ S_BRANCH %bb.4
+
+ bb.3:
+ successors: %bb.4
+ S_BRANCH %bb.4
+
+ bb.4:
+ $exec_lo = S_OR_B32 $exec_lo, %0, implicit-def $scc
+ S_ENDPGM 0
+...
+
+# An arbitrary EXEC write is not an SI_END_CF restore.
+---
+name: no_sink_wrong_exec_restore
+tracksRegLiveness: true
+isSSA: false
+machineFunctionInfo:
+ isEntryFunction: true
+ sgprForEXECCopy: '$sgpr105'
+body: |
+ ; CHECK-LABEL: name: no_sink_wrong_exec_restore
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
+ ; CHECK-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR [[DEF1]], [[DEF2]], [[DEF]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: $exec_lo = S_MOV_B32 0
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1, %bb.2
+ %3:vgpr_32 = IMPLICIT_DEF
+ %6:sgpr_64 = IMPLICIT_DEF
+ %9:vgpr_32 = IMPLICIT_DEF
+ %0:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
+ S_CBRANCH_EXECZ %bb.2, implicit $exec
+ S_BRANCH %bb.1
+
+ bb.1:
+ GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR %6, %9, %3, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+
+ bb.2:
+ $exec_lo = S_MOV_B32 0
+ S_ENDPGM 0
+...
+
+# Multiple DMAs retain their order without treating a later DMA's local
+# operand definition as a crossed dependency.
+---
+name: sink_multiple_dmas
+tracksRegLiveness: true
+isSSA: false
+machineFunctionInfo:
+ isEntryFunction: true
+ sgprForEXECCopy: '$sgpr105'
+body: |
+ ; CHECK-LABEL: name: sink_multiple_dmas
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
+ ; CHECK-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR [[DEF1]], [[DEF2]], [[DEF]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR [[DEF1]], [[V_MOV_B32_e32_]], [[DEF]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: $exec_lo = S_OR_B32 $exec_lo, [[COPY]], implicit-def $scc
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1, %bb.2
+ %3:vgpr_32 = IMPLICIT_DEF
+ %6:sgpr_64 = IMPLICIT_DEF
+ %9:vgpr_32 = IMPLICIT_DEF
+ %0:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
+ S_CBRANCH_EXECZ %bb.2, implicit $exec
+ S_BRANCH %bb.1
+
+ bb.1:
+ GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR %6, %9, %3, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ %11:vgpr_32 = V_MOV_B32_e32 4, implicit $exec
+ GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR %6, %11, %3, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+
+ bb.2:
+ $exec_lo = S_OR_B32 $exec_lo, %0, implicit-def $scc
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/sink-async-dma-out-of-execz.ll b/llvm/test/CodeGen/AMDGPU/sink-async-dma-out-of-execz.ll
index 5f9351e63b8b2..0a6f05db77baf 100644
--- a/llvm/test/CodeGen/AMDGPU/sink-async-dma-out-of-execz.ll
+++ b/llvm/test/CodeGen/AMDGPU/sink-async-dma-out-of-execz.ll
@@ -1,14 +1,15 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX1250 %s
-; SILowerControlFlow sinks async DMA out of s_cbranch_execz-guarded then-blocks
-; into the join-block, immediately before the s_or_b32 EXEC restore. This
-; makes ASYNCcnt deterministic regardless of whether the wave skipped the
-; block, which is required for correct waitcnt counting in software-pipelined
-; loops.
+; The SISinkAsyncDMA pass sinks async DMA out of s_cbranch_execz-guarded
+; then-blocks into the join-block, immediately before the s_or_b32 EXEC
+; restore, so every wave issues the DMA and the ASYNCcnt observed at the join
+; point no longer depends on whether the wave skipped the block. This is a
+; determinism optimization; correctness never relies on it firing.
-; Basic shape: async DMA sinks into %join right before s_or_b32 exec.
+; End-to-end: the async DMA sinks into %join right before the s_or_b32 EXEC
+; restore, and the block-local address def gets a header-side IMPLICIT_DEF.
define amdgpu_ps void @async_store_simple(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr, i32 %bound) {
; GFX1250-LABEL: async_store_simple:
; GFX1250: ; %bb.0: ; %entry
@@ -40,72 +41,6 @@ join:
ret void
}
-; SWP-style pattern: address compute lives inside the masked block, so the
-; sink inserts header-side IMPLICIT_DEFs for the DMA's address vregs to
-; satisfy SSA dominance on the EXECZ-skip path.
- at global_smem = external addrspace(3) global [0 x i8], align 16
-
-define amdgpu_kernel void @async_load_pipelined(ptr addrspace(1) readonly %src, i32 %M, i32 %N) {
-; GFX1250-LABEL: async_load_pipelined:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c nv
-; GFX1250-NEXT: v_and_b32_e32 v3, 0x7f, v0
-; GFX1250-NEXT: v_bfe_u32 v2, v0, 3, 7
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_cmp_gt_i32_e32 vcc_lo, s0, v3
-; GFX1250-NEXT: v_cmp_gt_i32_e64 s0, s1, v2
-; GFX1250-NEXT: s_and_b32 s1, vcc_lo, s0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_saveexec_b32 s0, s1
-; GFX1250-NEXT: ; implicit-def: $vgpr4
-; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1250-NEXT: s_cbranch_execz .LBB1_2
-; GFX1250-NEXT: ; %bb.1: ; %do_load
-; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 3, v3
-; GFX1250-NEXT: v_lshl_add_u32 v3, v3, 7, global_smem at abs32@lo
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
-; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 3, v2
-; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], v[4:5], v[0:1]
-; GFX1250-NEXT: v_lshl_add_u32 v4, v2, 4, v3
-; GFX1250-NEXT: .LBB1_2: ; %join
-; GFX1250-NEXT: global_load_async_to_lds_b128 v4, v[0:1], off
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT: s_wait_asynccnt 0x0
-; GFX1250-NEXT: s_endpgm
-entry:
- %tid = tail call i32 @llvm.amdgcn.workitem.id.x()
- %row = and i32 %tid, 127
- %col = lshr i32 %tid, 3
- %cmp_row = icmp slt i32 %row, %M
- %cmp_col = icmp slt i32 %col, %N
- %mask = and i1 %cmp_row, %cmp_col
- %lds_row_off = shl nuw nsw i32 %row, 7
- %lds_col_off = shl nuw nsw i32 %col, 4
- %lds_base = getelementptr inbounds nuw i8, ptr addrspace(3) @global_smem, i32 %lds_row_off
- %lds_addr = getelementptr inbounds nuw i8, ptr addrspace(3) %lds_base, i32 %lds_col_off
- %off_row = sext i32 %row to i64
- %src_row = getelementptr [8 x i8], ptr addrspace(1) %src, i64 %off_row
- %off_col = sext i32 %col to i64
- %src_addr = getelementptr [8 x i8], ptr addrspace(1) %src_row, i64 %off_col
- br i1 %mask, label %do_load, label %skip
-
-do_load:
- tail call void @llvm.amdgcn.global.load.async.to.lds.b128(ptr addrspace(1) %src_addr, ptr addrspace(3) %lds_addr, i32 0, i32 0)
- br label %join
-
-skip:
- br label %join
-
-join:
- tail call void @llvm.amdgcn.s.wait.asynccnt(i16 0)
- ret void
-}
-
; Negative test: an explicit s_wait_asynccnt inside the then-block disqualifies
; sinking. Skipping the wait on the EXECZ path would observe a different
; ASYNCcnt value, so the eligibility scan must bail and the branch must stay.
@@ -115,51 +50,12 @@ define amdgpu_ps void @async_load_then_block_has_waitcnt(ptr addrspace(1) inreg
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s2, exec_lo
; GFX1250-NEXT: v_cmpx_lt_i32_e64 s0, v1
-; GFX1250-NEXT: s_cbranch_execz .LBB2_2
-; GFX1250-NEXT: ; %bb.1: ; %do_load
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NEXT: global_load_async_to_lds_b128 v0, v1, s[0:1]
-; GFX1250-NEXT: s_wait_asynccnt 0x0
-; GFX1250-NEXT: .LBB2_2: ; %join
-; GFX1250-NEXT: s_endpgm
-entry:
- %tid = tail call i32 @llvm.amdgcn.workitem.id.x()
- %cmp = icmp slt i32 %tid, %bound
- br i1 %cmp, label %do_load, label %skip
-
-do_load:
- tail call void @llvm.amdgcn.global.load.async.to.lds.b128(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr, i32 0, i32 0)
- tail call void @llvm.amdgcn.s.wait.asynccnt(i16 0)
- br label %join
-
-skip:
- br label %join
-
-join:
- ret void
-}
-
-; Negative test: a non-invariant DS load reading from the LDS region the
-; async load writes to would read stale data if the DMA were sunk past it.
-; The mayLoad bail must fire.
-define amdgpu_ps void @async_load_then_block_has_aliasing_lds_load(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr, ptr addrspace(1) %out, i32 %bound) {
-; GFX1250-LABEL: async_load_then_block_has_aliasing_lds_load:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
-; GFX1250-NEXT: s_mov_b32 s2, exec_lo
-; GFX1250-NEXT: v_cmpx_lt_i32_e64 s0, v3
-; GFX1250-NEXT: s_cbranch_execz .LBB3_2
+; GFX1250-NEXT: s_cbranch_execz .LBB1_2
; GFX1250-NEXT: ; %bb.1: ; %do_load
; GFX1250-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-NEXT: global_load_async_to_lds_b128 v0, v1, s[0:1]
-; GFX1250-NEXT: ds_load_b32 v0, v0
-; GFX1250-NEXT: s_wait_dscnt 0x0
-; GFX1250-NEXT: global_store_b32 v[4:5], v0, off
-; GFX1250-NEXT: .LBB3_2: ; %join
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1250-NEXT: s_wait_asynccnt 0x0
+; GFX1250-NEXT: .LBB1_2: ; %join
; GFX1250-NEXT: s_endpgm
entry:
%tid = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -168,49 +64,12 @@ entry:
do_load:
tail call void @llvm.amdgcn.global.load.async.to.lds.b128(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr, i32 0, i32 0)
- %val = load i32, ptr addrspace(3) %laddr
- store i32 %val, ptr addrspace(1) %out
- br label %join
-
-skip:
- br label %join
-
-join:
tail call void @llvm.amdgcn.s.wait.asynccnt(i16 0)
- ret void
-}
-
-; Negative test: a regular store to the same global address the async load
-define amdgpu_ps void @async_load_then_block_has_aliasing_global_store(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr, i32 %bound) {
-; GFX1250-LABEL: async_load_then_block_has_aliasing_global_store:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_cmp_lt_i32_e32 vcc_lo, s0, v3
-; GFX1250-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1250-NEXT: s_cbranch_execz .LBB4_2
-; GFX1250-NEXT: ; %bb.1: ; %do_load
-; GFX1250-NEXT: global_load_async_to_lds_b128 v2, v[0:1], off
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
-; GFX1250-NEXT: .LBB4_2: ; %join
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT: s_wait_asynccnt 0x0
-; GFX1250-NEXT: s_endpgm
-entry:
- %tid = tail call i32 @llvm.amdgcn.workitem.id.x()
- %cmp = icmp slt i32 %tid, %bound
- br i1 %cmp, label %do_load, label %skip
-
-do_load:
- tail call void @llvm.amdgcn.global.load.async.to.lds.b128(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr, i32 0, i32 0)
- store i32 0, ptr addrspace(1) %gaddr
br label %join
skip:
br label %join
join:
- tail call void @llvm.amdgcn.s.wait.asynccnt(i16 0)
ret void
}
>From d406cf717897151473d47b5eb95071c9cdfcfd06 Mon Sep 17 00:00:00 2001
From: Vigneshwar <vjayakum at amd.com>
Date: Wed, 5 Aug 2026 23:23:26 -0500
Subject: [PATCH 3/3] changed to si_if
---
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 14 +-
llvm/lib/Target/AMDGPU/SISinkAsyncDMA.cpp | 238 ++++-----
llvm/lib/Target/AMDGPU/SISinkAsyncDMA.h | 12 +-
llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll | 7 +-
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 24 +-
.../test/CodeGen/AMDGPU/si-sink-async-dma.mir | 464 +++++++-----------
.../AMDGPU/sink-async-dma-out-of-execz.ll | 173 +++++--
7 files changed, 473 insertions(+), 459 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 0ec7f01e5ecf5..f5fb9429b68c1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -1825,7 +1825,6 @@ void GCNPassConfig::addFastRegAlloc() {
// TwoAddressInstructions, otherwise the processing of the tied operand of
// SI_ELSE will introduce a copy of the tied operand source after the else.
insertPass(&PHIEliminationID, &SILowerControlFlowLegacyID);
- insertPass(&SILowerControlFlowLegacyID, &SISinkAsyncDMALegacyID);
insertPass(&TwoAddressInstructionPassID, &SIWholeQuadModeID);
@@ -1833,8 +1832,10 @@ void GCNPassConfig::addFastRegAlloc() {
}
void GCNPassConfig::addPreRegAlloc() {
- if (getOptLevel() != CodeGenOptLevel::None)
+ if (getOptLevel() != CodeGenOptLevel::None) {
+ addPass(&SISinkAsyncDMALegacyID);
addPass(&AMDGPUPrepareAGPRAllocLegacyID);
+ }
}
void GCNPassConfig::addOptimizedRegAlloc() {
@@ -1852,7 +1853,6 @@ void GCNPassConfig::addOptimizedRegAlloc() {
// TwoAddressInstructions, otherwise the processing of the tied operand of
// SI_ELSE will introduce a copy of the tied operand source after the else.
insertPass(&PHIEliminationID, &SILowerControlFlowLegacyID);
- insertPass(&SILowerControlFlowLegacyID, &SISinkAsyncDMALegacyID);
if (EnableRewritePartialRegUses)
insertPass(&RenameIndependentSubregsID, &GCNRewritePartialRegUsesID);
@@ -2560,7 +2560,6 @@ void AMDGPUCodeGenPassBuilder::addMachineSSAOptimization(
Error AMDGPUCodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) const {
insertPass<PHIEliminationPass>(SILowerControlFlowPass());
- insertPass<PHIEliminationPass>(SISinkAsyncDMAPass());
insertPass<TwoAddressInstructionPass>(SIWholeQuadModePass());
@@ -2623,7 +2622,6 @@ Error AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(
// TwoAddressInstructions, otherwise the processing of the tied operand of
// SI_ELSE will introduce a copy of the tied operand source after the else.
insertPass<PHIEliminationPass>(SILowerControlFlowPass());
- insertPass<PHIEliminationPass>(SISinkAsyncDMAPass());
if (EnableRewritePartialRegUses)
insertPass<RenameIndependentSubregsPass>(GCNRewritePartialRegUsesPass());
@@ -2647,8 +2645,12 @@ Error AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(
}
void AMDGPUCodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) const {
- if (getOptLevel() != CodeGenOptLevel::None)
+ if (getOptLevel() != CodeGenOptLevel::None) {
+ // Still in SSA, which the PHI repair needs, and the last CFG change before
+ // SILowerControlFlow, which runs right after PHI elimination.
+ addMachineFunctionPass(SISinkAsyncDMAPass(), PMW);
addMachineFunctionPass(AMDGPUPrepareAGPRAllocPass(), PMW);
+ }
}
Expected<bool> AMDGPUCodeGenPassBuilder::addRegAssignAndRewriteOptimized(
diff --git a/llvm/lib/Target/AMDGPU/SISinkAsyncDMA.cpp b/llvm/lib/Target/AMDGPU/SISinkAsyncDMA.cpp
index 07c28c8717357..4a4ba7e55bb9c 100644
--- a/llvm/lib/Target/AMDGPU/SISinkAsyncDMA.cpp
+++ b/llvm/lib/Target/AMDGPU/SISinkAsyncDMA.cpp
@@ -8,35 +8,38 @@
//
/// \file
/// LLVM lowers a divergent branch around global_load_async_to_lds /
-/// global_store_async_from_lds with an S_CBRANCH_EXECZ. Fully-masked waves
-/// therefore skip the DMA entirely, which makes the ASYNCcnt observed at the
-/// join point depend on whether the wave took the branch. Software-pipelined
-/// kernels must then use conservative async waitcnts.
+/// global_store_async_from_lds with an S_CBRANCH_EXECZ, so fully-masked waves
+/// skip the DMA entirely and the ASYNCcnt observed at the join depends on
+/// whether the wave took the branch. Software-pipelined kernels then have to
+/// use a conservative async waitcnt.
///
-/// This pass sinks each async DMA out of such a then-block into the join
-/// block, immediately before the EXEC-mask restore (the S_OR that ends the
-/// control-flow region). EXEC at the sunk slot is the same masked value that
-/// guarded the then-block, so per-lane behavior is unchanged, but every wave
-/// now issues the DMA and ASYNCcnt becomes deterministic.
+/// This pass sinks each such DMA into the join, immediately before SI_ELSE or
+/// SI_END_CF:
///
-/// This runs right after SILowerControlFlow so the EXEC restore anchor exists,
-/// and before waitcnt insertion so the improved counts can be used.
+/// MBB MBB SI_IF sets EXEC to the then-block
+/// / \ / \ mask before the branch, so both
+/// ThenBB | ThenBB | edges carry it and per-lane
+/// [DMA] | ==> \ / behavior is unchanged. But every
+/// \ / JoinBB wave now issues the DMA, so
+/// JoinBB [DMA] ASYNCcnt at the join no longer
+/// [SI_END_CF] | depends on the branch.
+/// [SI_END_CF]
///
-/// This is determinism-only: it never relaxes or rewrites a wait (an
-/// s_wait_asynccnt 0 stays 0). Correctness never depends on the transform
-/// firing; with the pass disabled the code is still correct, just conservative.
+/// The join is split so that SI_END_CF starts a block of its own, because
+/// SILowerControlFlow emits the EXEC restore at the top of the block holding
+/// it, which would otherwise place it above the sunk DMAs.
+
//
//===----------------------------------------------------------------------===//
#include "SISinkAsyncDMA.h"
#include "AMDGPU.h"
-#include "AMDGPULaneMaskUtils.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIInstrInfo.h"
-#include "llvm/ADT/SmallSet.h"
-#include "llvm/CodeGen/LiveVariables.h"
+#include "llvm/ADT/DenseMap.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineSSAUpdater.h"
using namespace llvm;
@@ -48,16 +51,10 @@ class SISinkAsyncDMA {
const SIInstrInfo *TII = nullptr;
const SIRegisterInfo *TRI = nullptr;
MachineRegisterInfo *MRI = nullptr;
- LiveVariables *LV = nullptr;
- const AMDGPU::LaneMaskConstants &LMC;
bool sinkFromBlock(MachineBasicBlock &MBB);
public:
- SISinkAsyncDMA(const GCNSubtarget *ST, LiveVariables *LV)
- : TII(ST->getInstrInfo()), TRI(&TII->getRegisterInfo()), LV(LV),
- LMC(AMDGPU::LaneMaskConstants::get(*ST)) {}
-
bool run(MachineFunction &MF);
};
@@ -73,11 +70,12 @@ class SISinkAsyncDMALegacy : public MachineFunctionPass {
return "SI sink async DMA out of execz then-blocks";
}
- void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.setPreservesCFG();
- AU.addUsedIfAvailable<LiveVariablesWrapperPass>();
- AU.addPreserved<LiveVariablesWrapperPass>();
- MachineFunctionPass::getAnalysisUsage(AU);
+ MachineFunctionProperties getRequiredProperties() const override {
+ return MachineFunctionProperties().setIsSSA();
+ }
+
+ MachineFunctionProperties getClearedProperties() const override {
+ return MachineFunctionProperties().setNoPHIs();
}
};
@@ -90,136 +88,122 @@ INITIALIZE_PASS(SISinkAsyncDMALegacy, DEBUG_TYPE,
char &llvm::SISinkAsyncDMALegacyID = SISinkAsyncDMALegacy::ID;
-/// Return the S_OR EXEC restore at the top of \p JoinBB, if present.
-static MachineInstr *findExecRestore(MachineBasicBlock &JoinBB,
- const AMDGPU::LaneMaskConstants &LMC) {
- auto I = JoinBB.getFirstNonDebugInstr();
- return I != JoinBB.end() && I->getOpcode() == LMC.OrOpc &&
- I->getOperand(0).getReg() == LMC.ExecReg &&
- I->getOperand(1).getReg() == LMC.ExecReg
- ? &*I
- : nullptr;
-}
-
static bool isAsyncDMA(const MachineInstr &MI) {
return SIInstrInfo::isLDSDMA(MI) && SIInstrInfo::usesASYNC_CNT(MI);
}
+static bool isAsyncMarker(const MachineInstr &MI) {
+ return MI.getOpcode() == AMDGPU::ASYNCMARK ||
+ MI.getOpcode() == AMDGPU::WAIT_ASYNCMARK;
+}
+
bool SISinkAsyncDMA::sinkFromBlock(MachineBasicBlock &MBB) {
if (MBB.succ_size() != 2)
return false;
- MachineBasicBlock *TBB = nullptr;
- MachineBasicBlock *FBB = nullptr;
- SmallVector<MachineOperand, 4> Cond;
- if (TII->analyzeBranch(MBB, TBB, FBB, Cond) || !TBB || Cond.empty())
+ // A region head ends in SI_IF or SI_ELSE ($dst, $cond, $target), which define
+ // the mask the region restores in $dst and the join block in $target.
+ auto ControlMI = MBB.getFirstTerminator();
+ if (ControlMI == MBB.end() || (ControlMI->getOpcode() != AMDGPU::SI_IF &&
+ ControlMI->getOpcode() != AMDGPU::SI_ELSE))
return false;
- auto CondBr = find_if(MBB.terminators(), [](const MachineInstr &MI) {
- return MI.isConditionalBranch();
- });
- if (CondBr == MBB.terminators().end() ||
- CondBr->getOpcode() != AMDGPU::S_CBRANCH_EXECZ)
+ Register SavedExec = ControlMI->getOperand(0).getReg();
+ MachineBasicBlock *JoinBB = ControlMI->getOperand(2).getMBB();
+
+ if (!MBB.isSuccessor(JoinBB) || JoinBB->pred_size() != 2)
return false;
- MachineBasicBlock *JoinBB = TBB;
- MachineBasicBlock *S0 = *MBB.succ_begin();
- MachineBasicBlock *S1 = *std::next(MBB.succ_begin());
- MachineBasicBlock *ThenBB = (S0 == JoinBB) ? S1 : S0;
- if (ThenBB == JoinBB || ThenBB->succ_size() != 1 ||
- *ThenBB->succ_begin() != JoinBB)
+ auto ThenIt = find_if(MBB.successors(),
+ [JoinBB](MachineBasicBlock *S) { return S != JoinBB; });
+ if (ThenIt == MBB.succ_end() || (*ThenIt)->getSingleSuccessor() != JoinBB)
return false;
+ MachineBasicBlock *ThenBB = *ThenIt;
- // A third incoming edge could reach the DMA under an unrelated EXEC.
- if (JoinBB->pred_size() != 2)
+ auto Boundary = JoinBB->getFirstNonPHI();
+ while (Boundary != JoinBB->end() && Boundary->isMetaInstruction())
+ ++Boundary;
+ if (Boundary == JoinBB->end())
return false;
- MachineInstr *ExecRestore = findExecRestore(*JoinBB, LMC);
- if (!ExecRestore)
+ // The boundary must consume the mask this region saved ($saved for SI_END_CF,
+ // $src for a chained SI_ELSE), otherwise it closes a different region.
+ bool IsEndCF = Boundary->getOpcode() == AMDGPU::SI_END_CF &&
+ Boundary->getOperand(0).getReg() == SavedExec;
+ bool IsElse = ControlMI->getOpcode() == AMDGPU::SI_IF &&
+ Boundary->getOpcode() == AMDGPU::SI_ELSE &&
+ Boundary->getOperand(1).getReg() == SavedExec;
+ if (!IsEndCF && !IsElse)
return false;
+ // Scan bottom-up so everything a DMA moves across is already accumulated when
+ // the DMA is reached. Instructions above the topmost DMA are never crossed,
+ // so an unsafe one only matters once a DMA turns up above it.
SmallVector<MachineInstr *, 4> ToSink;
- for (MachineInstr &TMI : *ThenBB) {
- if (TMI.isMetaInstruction() || TMI.isTerminator())
- continue;
- if (TII->hasUnwantedEffectsWhenEXECEmpty(TMI))
+ bool CrossedUnsafe = false;
+ bool CrossedM0Write = false;
+
+ for (MachineInstr &TMI : reverse(*ThenBB)) {
+ if (isAsyncMarker(TMI))
return false;
+ if (TMI.isMetaInstruction())
+ continue;
+
if (isAsyncDMA(TMI)) {
+ // A cluster load takes its mask from M0.
+ if (CrossedUnsafe ||
+ (CrossedM0Write && TMI.readsRegister(AMDGPU::M0, TRI)))
+ return false;
ToSink.push_back(&TMI);
continue;
}
- if (TMI.modifiesRegister(AMDGPU::EXEC, TRI) ||
- TMI.hasUnmodeledSideEffects())
- return false;
-
- if (ToSink.empty())
- continue;
- if (TMI.mayStore())
- return false;
- if (TMI.mayLoad() && !TMI.isDereferenceableInvariantLoad())
- return false;
- // Reject dependencies with earlier DMAs that would cross this instruction.
- for (const MachineOperand &MO : TMI.operands()) {
- if (!MO.isReg() || !MO.getReg() || (!MO.isDef() && !MO.readsReg()) ||
- TRI->regsOverlap(MO.getReg(), AMDGPU::EXEC))
- continue;
- if (any_of(ToSink, [&](const MachineInstr *DmaMI) {
- return DmaMI->readsRegister(MO.getReg(), TRI) ||
- DmaMI->modifiesRegister(MO.getReg(), TRI);
- }))
- return false;
- }
+ CrossedUnsafe |= TII->hasUnwantedEffectsWhenEXECEmpty(TMI) ||
+ TMI.modifiesRegister(AMDGPU::EXEC, TRI) ||
+ TMI.hasUnmodeledSideEffects() || TMI.mayStore() ||
+ (TMI.mayLoad() && !TMI.isDereferenceableInvariantLoad());
+ CrossedM0Write |= TMI.modifiesRegister(AMDGPU::M0, TRI);
}
if (ToSink.empty())
return false;
- SmallSet<Register, 4> NeedsImpDef;
- SmallSet<Register, 8> LiveThroughThen;
- for (const MachineInstr *DmaMI : ToSink) {
- for (const MachineOperand &MO : DmaMI->uses()) {
+ std::reverse(ToSink.begin(), ToSink.end());
+
+ MachineSSAUpdater Updater(*MBB.getParent());
+ SmallDenseMap<Register, Register, 4> MergedRegs;
+ for (MachineInstr *DmaMI : ToSink) {
+ for (MachineOperand &MO : DmaMI->uses()) {
if (!MO.isReg() || !MO.readsReg() || !MO.getReg().isVirtual())
continue;
Register Reg = MO.getReg();
- MachineInstr *Def = MRI->getUniqueVRegDef(Reg);
- if (!Def)
- return false;
- if (Def->getParent() == ThenBB) {
- if (!isAsyncDMA(*Def))
- NeedsImpDef.insert(Reg);
- } else {
- LiveThroughThen.insert(Reg);
+ if (MRI->getVRegDef(Reg)->getParent() != ThenBB)
+ continue;
+ Register &Merged = MergedRegs[Reg];
+ if (!Merged) {
+ Updater.Initialize(Reg);
+ Updater.AddAvailableValue(ThenBB, Reg);
+ Merged = Updater.GetValueInMiddleOfBlock(JoinBB);
}
+ MO.setReg(Merged);
}
- }
-
- auto FirstTerm = MBB.getFirstTerminator();
- for (Register Reg : NeedsImpDef)
- BuildMI(MBB, FirstTerm, FirstTerm->getDebugLoc(),
- TII->get(TargetOpcode::IMPLICIT_DEF), Reg);
- for (MachineInstr *DmaMI : ToSink) {
LLVM_DEBUG(dbgs() << "Sinking async DMA out of execz then-block: "
<< *DmaMI);
- DmaMI->moveBefore(ExecRestore);
- for (const MachineOperand &MO : DmaMI->uses()) {
- if (!MO.isReg() || !MO.readsReg())
- continue;
- Register Reg = MO.getReg();
- if (Reg.isPhysical() && Reg != LMC.ExecReg)
- JoinBB->addLiveIn(Reg);
- }
+ DmaMI->moveBefore(&*Boundary);
}
- if (LV)
- for (Register Reg : LiveThroughThen)
- LV->getVarInfo(Reg).AliveBlocks.set(ThenBB->getNumber());
+ JoinBB->splitAt(*ToSink.back(), /*UpdateLiveIns=*/true);
- JoinBB->sortUniqueLiveIns();
return true;
}
bool SISinkAsyncDMA::run(MachineFunction &MF) {
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ if (!ST.hasAsynccnt())
+ return false;
+
+ TII = ST.getInstrInfo();
+ TRI = &TII->getRegisterInfo();
MRI = &MF.getRegInfo();
bool Changed = false;
@@ -230,24 +214,16 @@ bool SISinkAsyncDMA::run(MachineFunction &MF) {
}
bool SISinkAsyncDMALegacy::runOnMachineFunction(MachineFunction &MF) {
- const GCNSubtarget *ST = &MF.getSubtarget<GCNSubtarget>();
- auto *LVWrapper = getAnalysisIfAvailable<LiveVariablesWrapperPass>();
- LiveVariables *LV = LVWrapper ? &LVWrapper->getLV() : nullptr;
- return SISinkAsyncDMA(ST, LV).run(MF);
-}
+ if (skipFunction(MF.getFunction()))
+ return false;
-PreservedAnalyses
-SISinkAsyncDMAPass::run(MachineFunction &MF,
- MachineFunctionAnalysisManager &MFAM) {
- const GCNSubtarget *ST = &MF.getSubtarget<GCNSubtarget>();
- LiveVariables *LV = MFAM.getCachedResult<LiveVariablesAnalysis>(MF);
+ return SISinkAsyncDMA().run(MF);
+}
- bool Changed = SISinkAsyncDMA(ST, LV).run(MF);
- if (!Changed)
- return PreservedAnalyses::all();
+PreservedAnalyses SISinkAsyncDMAPass::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &) {
+ MFPropsModifier _(*this, MF);
- auto PA = getMachineFunctionPassPreservedAnalyses();
- PA.preserveSet<CFGAnalyses>();
- PA.preserve<LiveVariablesAnalysis>();
- return PA;
+ return SISinkAsyncDMA().run(MF) ? getMachineFunctionPassPreservedAnalyses()
+ : PreservedAnalyses::all();
}
diff --git a/llvm/lib/Target/AMDGPU/SISinkAsyncDMA.h b/llvm/lib/Target/AMDGPU/SISinkAsyncDMA.h
index 93189b6a69399..ae6335ede2352 100644
--- a/llvm/lib/Target/AMDGPU/SISinkAsyncDMA.h
+++ b/llvm/lib/Target/AMDGPU/SISinkAsyncDMA.h
@@ -1,4 +1,4 @@
-//===- SISinkAsyncDMA.h -----------------------------------------*- C++- *-===//
+//===- SISinkAsyncDMA.h -----------------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -12,10 +12,18 @@
#include "llvm/CodeGen/MachinePassManager.h"
namespace llvm {
-class SISinkAsyncDMAPass : public PassInfoMixin<SISinkAsyncDMAPass> {
+class SISinkAsyncDMAPass : public OptionalPassInfoMixin<SISinkAsyncDMAPass> {
public:
PreservedAnalyses run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM);
+
+ MachineFunctionProperties getRequiredProperties() const {
+ return MachineFunctionProperties().setIsSSA();
+ }
+
+ MachineFunctionProperties getClearedProperties() const {
+ return MachineFunctionProperties().setNoPHIs();
+ }
};
} // namespace llvm
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
index c772da55dbdfa..bcee57474e181 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
@@ -70,7 +70,6 @@
; GCN-O0-NEXT: reg-usage-propagation
; GCN-O0-NEXT: phi-node-elimination
; GCN-O0-NEXT: si-lower-control-flow
-; GCN-O0-NEXT: si-sink-async-dma
; GCN-O0-NEXT: two-address-instruction
; GCN-O0-NEXT: si-wqm
; GCN-O0-NEXT: amdgpu-pre-ra-long-branch-reg
@@ -104,7 +103,7 @@
; GCN-O0-NEXT: amdgpu-preload-kern-arg-prolog
; GCN-O0-NEXT: stack-frame-layout
; GCN-O0-NEXT: amdgpu-asm-printer
-; GCN-O0-NEXT: free-machine-function
+; GCN-O0-NEXT: free-machine-function
; GCN-O0-NEXT: amdgpu-asm-printer-end
; GCN-O2: require<MachineModuleAnalysis>
@@ -216,6 +215,7 @@
; GCN-O2-NEXT: function
; GCN-O2-NEXT: machine-function
; GCN-O2-NEXT: reg-usage-propagation
+; GCN-O2-NEXT: si-sink-async-dma
; GCN-O2-NEXT: amdgpu-prepare-agpr-alloc
; GCN-O2-NEXT: detect-dead-lanes
; GCN-O2-NEXT: dead-mi-elimination
@@ -227,7 +227,6 @@
; GCN-O2-NEXT: require<machine-loops>
; GCN-O2-NEXT: phi-node-elimination
; GCN-O2-NEXT: si-lower-control-flow
-; GCN-O2-NEXT: si-sink-async-dma
; GCN-O2-NEXT: two-address-instruction
; GCN-O2-NEXT: register-coalescer
; GCN-O2-NEXT: rename-independent-subregs
@@ -404,6 +403,7 @@
; GCN-O3-NEXT: function
; GCN-O3-NEXT: machine-function
; GCN-O3-NEXT: reg-usage-propagation
+; GCN-O3-NEXT: si-sink-async-dma
; GCN-O3-NEXT: amdgpu-prepare-agpr-alloc
; GCN-O3-NEXT: detect-dead-lanes
; GCN-O3-NEXT: dead-mi-elimination
@@ -415,7 +415,6 @@
; GCN-O3-NEXT: require<machine-loops>
; GCN-O3-NEXT: phi-node-elimination
; GCN-O3-NEXT: si-lower-control-flow
-; GCN-O3-NEXT: si-sink-async-dma
; GCN-O3-NEXT: two-address-instruction
; GCN-O3-NEXT: register-coalescer
; GCN-O3-NEXT: rename-independent-subregs
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index 3ca35e325d881..e93e8f1185b58 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -115,7 +115,6 @@
; GCN-O0-NEXT: Register Usage Information Propagation
; GCN-O0-NEXT: Eliminate PHI nodes for register allocation
; GCN-O0-NEXT: SI Lower control flow pseudo instructions
-; GCN-O0-NEXT: SI sink async DMA out of execz then-blocks
; GCN-O0-NEXT: Two-Address instruction pass
; GCN-O0-NEXT: MachineDominator Tree Construction
; GCN-O0-NEXT: Slot index numbering
@@ -356,6 +355,7 @@
; GCN-O1-NEXT: Remove dead machine instructions
; GCN-O1-NEXT: SI Shrink Instructions
; GCN-O1-NEXT: Register Usage Information Propagation
+; GCN-O1-NEXT: SI sink async DMA out of execz then-blocks
; GCN-O1-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O1-NEXT: Detect Dead Lanes
; GCN-O1-NEXT: Remove dead machine instructions
@@ -364,17 +364,19 @@
; GCN-O1-NEXT: Remove unreachable machine basic blocks
; GCN-O1-NEXT: Live Variable Analysis
; GCN-O1-NEXT: MachineDominator Tree Construction
+; GCN-O1-NEXT: Machine Natural Loop Construction
; GCN-O1-NEXT: SI Optimize VGPR LiveRange
; GCN-O1-NEXT: Eliminate PHI nodes for register allocation
; GCN-O1-NEXT: SI Lower control flow pseudo instructions
-; GCN-O1-NEXT: SI sink async DMA out of execz then-blocks
; GCN-O1-NEXT: Two-Address instruction pass
; GCN-O1-NEXT: Slot index numbering
; GCN-O1-NEXT: Live Interval Analysis
; GCN-O1-NEXT: Machine Natural Loop Construction
+; GCN-O1-NEXT: Machine Register Class Info Analysis
; GCN-O1-NEXT: Register Coalescer
; GCN-O1-NEXT: Rename Disconnected Subregister Components
; GCN-O1-NEXT: Rewrite Partial Register Uses
+; GCN-O1-NEXT: Machine Block Frequency Analysis
; GCN-O1-NEXT: Machine Instruction Scheduler
; GCN-O1-NEXT: SI Whole Quad Mode
; GCN-O1-NEXT: SI optimize exec mask operations pre-RA
@@ -684,6 +686,7 @@
; GCN-O1-OPTS-NEXT: Remove dead machine instructions
; GCN-O1-OPTS-NEXT: SI Shrink Instructions
; GCN-O1-OPTS-NEXT: Register Usage Information Propagation
+; GCN-O1-OPTS-NEXT: SI sink async DMA out of execz then-blocks
; GCN-O1-OPTS-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O1-OPTS-NEXT: Detect Dead Lanes
; GCN-O1-OPTS-NEXT: Remove dead machine instructions
@@ -691,17 +694,20 @@
; GCN-O1-OPTS-NEXT: Process Implicit Definitions
; GCN-O1-OPTS-NEXT: Remove unreachable machine basic blocks
; GCN-O1-OPTS-NEXT: Live Variable Analysis
+; GCN-O1-OPTS-NEXT: MachineDominator Tree Construction
+; GCN-O1-OPTS-NEXT: Machine Natural Loop Construction
; GCN-O1-OPTS-NEXT: SI Optimize VGPR LiveRange
; GCN-O1-OPTS-NEXT: Eliminate PHI nodes for register allocation
; GCN-O1-OPTS-NEXT: SI Lower control flow pseudo instructions
-; GCN-O1-OPTS-NEXT: SI sink async DMA out of execz then-blocks
; GCN-O1-OPTS-NEXT: Two-Address instruction pass
; GCN-O1-OPTS-NEXT: Slot index numbering
; GCN-O1-OPTS-NEXT: Live Interval Analysis
; GCN-O1-OPTS-NEXT: Machine Natural Loop Construction
+; GCN-O1-OPTS-NEXT: Machine Register Class Info Analysis
; GCN-O1-OPTS-NEXT: Register Coalescer
; GCN-O1-OPTS-NEXT: Rename Disconnected Subregister Components
; GCN-O1-OPTS-NEXT: Rewrite Partial Register Uses
+; GCN-O1-OPTS-NEXT: Machine Block Frequency Analysis
; GCN-O1-OPTS-NEXT: Machine Instruction Scheduler
; GCN-O1-OPTS-NEXT: AMDGPU Pre-RA optimizations
; GCN-O1-OPTS-NEXT: SI Whole Quad Mode
@@ -1016,6 +1022,7 @@
; GCN-O2-NEXT: Remove dead machine instructions
; GCN-O2-NEXT: SI Shrink Instructions
; GCN-O2-NEXT: Register Usage Information Propagation
+; GCN-O2-NEXT: SI sink async DMA out of execz then-blocks
; GCN-O2-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O2-NEXT: Detect Dead Lanes
; GCN-O2-NEXT: Remove dead machine instructions
@@ -1023,17 +1030,20 @@
; GCN-O2-NEXT: Process Implicit Definitions
; GCN-O2-NEXT: Remove unreachable machine basic blocks
; GCN-O2-NEXT: Live Variable Analysis
+; GCN-O2-NEXT: MachineDominator Tree Construction
+; GCN-O2-NEXT: Machine Natural Loop Construction
; GCN-O2-NEXT: SI Optimize VGPR LiveRange
; GCN-O2-NEXT: Eliminate PHI nodes for register allocation
; GCN-O2-NEXT: SI Lower control flow pseudo instructions
-; GCN-O2-NEXT: SI sink async DMA out of execz then-blocks
; GCN-O2-NEXT: Two-Address instruction pass
; GCN-O2-NEXT: Slot index numbering
; GCN-O2-NEXT: Live Interval Analysis
; GCN-O2-NEXT: Machine Natural Loop Construction
+; GCN-O2-NEXT: Machine Register Class Info Analysis
; GCN-O2-NEXT: Register Coalescer
; GCN-O2-NEXT: Rename Disconnected Subregister Components
; GCN-O2-NEXT: Rewrite Partial Register Uses
+; GCN-O2-NEXT: Machine Block Frequency Analysis
; GCN-O2-NEXT: Machine Instruction Scheduler
; GCN-O2-NEXT: AMDGPU Pre-RA optimizations
; GCN-O2-NEXT: SI Whole Quad Mode
@@ -1364,6 +1374,7 @@
; GCN-O3-NEXT: Remove dead machine instructions
; GCN-O3-NEXT: SI Shrink Instructions
; GCN-O3-NEXT: Register Usage Information Propagation
+; GCN-O3-NEXT: SI sink async DMA out of execz then-blocks
; GCN-O3-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O3-NEXT: Detect Dead Lanes
; GCN-O3-NEXT: Remove dead machine instructions
@@ -1371,17 +1382,20 @@
; GCN-O3-NEXT: Process Implicit Definitions
; GCN-O3-NEXT: Remove unreachable machine basic blocks
; GCN-O3-NEXT: Live Variable Analysis
+; GCN-O3-NEXT: MachineDominator Tree Construction
+; GCN-O3-NEXT: Machine Natural Loop Construction
; GCN-O3-NEXT: SI Optimize VGPR LiveRange
; GCN-O3-NEXT: Eliminate PHI nodes for register allocation
; GCN-O3-NEXT: SI Lower control flow pseudo instructions
-; GCN-O3-NEXT: SI sink async DMA out of execz then-blocks
; GCN-O3-NEXT: Two-Address instruction pass
; GCN-O3-NEXT: Slot index numbering
; GCN-O3-NEXT: Live Interval Analysis
; GCN-O3-NEXT: Machine Natural Loop Construction
+; GCN-O3-NEXT: Machine Register Class Info Analysis
; GCN-O3-NEXT: Register Coalescer
; GCN-O3-NEXT: Rename Disconnected Subregister Components
; GCN-O3-NEXT: Rewrite Partial Register Uses
+; GCN-O3-NEXT: Machine Block Frequency Analysis
; GCN-O3-NEXT: Machine Instruction Scheduler
; GCN-O3-NEXT: AMDGPU Pre-RA optimizations
; GCN-O3-NEXT: SI Whole Quad Mode
diff --git a/llvm/test/CodeGen/AMDGPU/si-sink-async-dma.mir b/llvm/test/CodeGen/AMDGPU/si-sink-async-dma.mir
index cac538c461a59..4951021410903 100644
--- a/llvm/test/CodeGen/AMDGPU/si-sink-async-dma.mir
+++ b/llvm/test/CodeGen/AMDGPU/si-sink-async-dma.mir
@@ -1,444 +1,360 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=si-sink-async-dma -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -passes=si-sink-async-dma -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu12.50 -passes=si-sink-async-dma -verify-machineinstrs -o - %s | FileCheck %s
-# Sink an async store immediately before the EXEC restore.
---
-name: sink_async_store
+name: sink_shared_then_local_operands
tracksRegLiveness: true
-isSSA: false
-machineFunctionInfo:
- isEntryFunction: true
- sgprForEXECCopy: '$sgpr105'
body: |
- ; CHECK-LABEL: name: sink_async_store
+ ; CHECK-LABEL: name: sink_shared_then_local_operands
; CHECK: bb.0:
; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
; CHECK-NEXT: [[DEF1:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
- ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CHECK-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF4:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[SI_IF:%[0-9]+]]:sreg_32 = SI_IF [[DEF2]], %bb.2, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[DEF1:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
- ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD_SADDR [[DEF3]], [[DEF3]], [[DEF4]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[DEF5:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B32_e32_]]
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: liveins: $asynccnt, $exec
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR killed [[DEF1]], killed [[DEF2]], killed [[DEF]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
- ; CHECK-NEXT: $exec_lo = S_OR_B32 $exec_lo, killed [[COPY]], implicit-def $scc
- ; CHECK-NEXT: S_ENDPGM 0
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[DEF]], %bb.0, [[V_MOV_B32_e32_]], %bb.1
+ ; CHECK-NEXT: [[PHI1:%[0-9]+]]:sgpr_64 = PHI [[DEF1]], %bb.0, [[DEF5]], %bb.1
+ ; CHECK-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR [[PHI1]], [[PHI]], [[DEF3]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR [[PHI1]], [[PHI]], [[DEF3]], 16, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
bb.0:
successors: %bb.1, %bb.2
- %3:vgpr_32 = IMPLICIT_DEF
- %0:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
- S_CBRANCH_EXECZ %bb.2, implicit $exec
+
+ %0:sreg_32 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ %2:sgpr_64 = IMPLICIT_DEF
+ %3:sreg_32 = SI_IF %0, %bb.2, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
S_BRANCH %bb.1
bb.1:
- %6:sgpr_64 = IMPLICIT_DEF
- %9:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR killed %6, killed %9, killed %3, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ successors: %bb.2
+
+ GLOBAL_STORE_DWORD_SADDR %1, %1, %2, 0, 0, implicit $exec
+ %4:sgpr_64 = IMPLICIT_DEF
+ %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR %4, %5, %1, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR %4, %5, %1, 16, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ %6:vgpr_32 = COPY %5
bb.2:
- $exec_lo = S_OR_B32 $exec_lo, killed %0, implicit-def $scc
- S_ENDPGM 0
+ SI_END_CF %3, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
...
-# Any non-DMA store prevents sinking because the pass has no alias information.
---
-name: no_sink_other_store
+name: sink_if_then_else
tracksRegLiveness: true
-isSSA: false
-machineFunctionInfo:
- isEntryFunction: true
- sgprForEXECCopy: '$sgpr105'
body: |
- ; CHECK-LABEL: name: no_sink_other_store
+ ; CHECK-LABEL: name: sink_if_then_else
; CHECK: bb.0:
; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
- ; CHECK-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[SI_IF:%[0-9]+]]:sreg_32 = SI_IF [[DEF]], %bb.2, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed [[DEF]], [[DEF1]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
- ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: GLOBAL_STORE_DWORD killed [[DEF1]], killed [[V_MOV_B32_e32_]], 0, 0, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: $exec_lo = S_OR_B32 $exec_lo, killed [[COPY]], implicit-def $scc
- ; CHECK-NEXT: S_ENDPGM 0
- bb.0:
- successors: %bb.1, %bb.2
- %3:vgpr_32 = IMPLICIT_DEF
- %12:vreg_64_align2 = IMPLICIT_DEF
- %0:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
- S_CBRANCH_EXECZ %bb.2, implicit $exec
- S_BRANCH %bb.1
-
- bb.1:
- GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed %3, %12, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
- %9:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- GLOBAL_STORE_DWORD killed %12, killed %9, 0, 0, implicit $exec
-
- bb.2:
- $exec_lo = S_OR_B32 $exec_lo, killed %0, implicit-def $scc
- S_ENDPGM 0
-...
-
-# Sink an async load and repair its block-local address definition.
----
-name: sink_async_load
-tracksRegLiveness: true
-isSSA: false
-machineFunctionInfo:
- isEntryFunction: true
- sgprForEXECCopy: '$sgpr105'
-body: |
- ; CHECK-LABEL: name: sink_async_load
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: successors: %bb.5(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
- ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
- ; CHECK-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR [[DEF2]], [[DEF1]], [[DEF1]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.4(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[SI_ELSE:%[0-9]+]]:sreg_32 = SI_ELSE [[SI_IF]], %bb.4, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.3
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: liveins: $asynccnt, $exec
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_BRANCH %bb.4
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed [[DEF]], killed [[DEF1]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
- ; CHECK-NEXT: $exec_lo = S_OR_B32 $exec_lo, killed [[COPY]], implicit-def $scc
- ; CHECK-NEXT: S_ENDPGM 0
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.6(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR [[DEF2]], [[DEF1]], [[DEF1]], 16, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.6:
+ ; CHECK-NEXT: SI_END_CF [[SI_ELSE]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
bb.0:
successors: %bb.1, %bb.2
- %3:vgpr_32 = IMPLICIT_DEF
- %0:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
- S_CBRANCH_EXECZ %bb.2, implicit $exec
+
+ %0:sreg_32 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ %2:sgpr_64 = IMPLICIT_DEF
+ %3:sreg_32 = SI_IF %0, %bb.2, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
S_BRANCH %bb.1
bb.1:
- %12:vreg_64_align2 = IMPLICIT_DEF
- GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed %3, killed %12, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ successors: %bb.2
+
+ GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR %2, %1, %1, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ S_BRANCH %bb.2
bb.2:
- $exec_lo = S_OR_B32 $exec_lo, killed %0, implicit-def $scc
- S_ENDPGM 0
+ successors: %bb.3, %bb.4
+
+ %4:sreg_32 = SI_ELSE %3, %bb.4, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+ S_BRANCH %bb.3
+
+ bb.3:
+ successors: %bb.4
+
+ GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR %2, %1, %1, 16, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ S_BRANCH %bb.4
+
+ bb.4:
+ SI_END_CF %4, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
...
-# An ASYNC_CNT barrier is not an LDS DMA candidate.
---
-name: no_sink_async_barrier_arrive
+name: no_sink_store_after_dma
tracksRegLiveness: true
-isSSA: false
-machineFunctionInfo:
- isEntryFunction: true
- sgprForEXECCopy: '$sgpr105'
body: |
- ; CHECK-LABEL: name: no_sink_async_barrier_arrive
+ ; CHECK-LABEL: name: no_sink_store_after_dma
; CHECK: bb.0:
; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
- ; CHECK-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[SI_IF:%[0-9]+]]:sreg_32 = SI_IF [[DEF]], %bb.2, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 killed [[DEF]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128_SADDR [[DEF1]], [[DEF2]], [[DEF1]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD_SADDR [[DEF1]], [[DEF1]], [[DEF2]], 0, 0, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: $exec_lo = S_OR_B32 $exec_lo, killed [[COPY]], implicit-def $scc
- ; CHECK-NEXT: S_ENDPGM 0
+ ; CHECK-NEXT: SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
bb.0:
successors: %bb.1, %bb.2
- %2:vgpr_32 = IMPLICIT_DEF
- %0:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
- S_CBRANCH_EXECZ %bb.2, implicit $exec
+
+ %0:sreg_32 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ %2:sgpr_64 = IMPLICIT_DEF
+ %3:sreg_32 = SI_IF %0, %bb.2, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
S_BRANCH %bb.1
bb.1:
- DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 killed %2, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ successors: %bb.2
+
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128_SADDR %1, %2, %1, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ GLOBAL_STORE_DWORD_SADDR %1, %1, %2, 0, 0, implicit $exec
bb.2:
- $exec_lo = S_OR_B32 $exec_lo, killed %0, implicit-def $scc
- S_ENDPGM 0
+ SI_END_CF %3, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
...
-# %12 has multiple defs, so the pass cannot add a header IMPLICIT_DEF that makes
-# its sunk use well-formed on the skip path.
---
-name: no_sink_multiple_defs
+name: no_sink_m0_write_after_cluster_load
tracksRegLiveness: true
-isSSA: false
-machineFunctionInfo:
- isEntryFunction: true
- sgprForEXECCopy: '$sgpr105'
body: |
- ; CHECK-LABEL: name: no_sink_multiple_defs
+ ; CHECK-LABEL: name: no_sink_m0_write_after_cluster_load
; CHECK: bb.0:
; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
- ; CHECK-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[SI_IF:%[0-9]+]]:sreg_32 = SI_IF [[DEF]], %bb.2, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
- ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
- ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed [[DEF]], killed [[DEF1]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: $m0 = S_MOV_B32 0
+ ; CHECK-NEXT: CLUSTER_LOAD_ASYNC_TO_LDS_B32_SADDR [[DEF1]], [[DEF2]], [[DEF1]], 0, 0, implicit-def dead $asynccnt, implicit $m0, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: $m0 = S_MOV_B32 1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: $exec_lo = S_OR_B32 $exec_lo, killed [[COPY]], implicit-def $scc
- ; CHECK-NEXT: S_ENDPGM 0
+ ; CHECK-NEXT: SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
bb.0:
successors: %bb.1, %bb.2
- %3:vgpr_32 = IMPLICIT_DEF
- %0:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
- S_CBRANCH_EXECZ %bb.2, implicit $exec
+
+ %0:sreg_32 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ %2:sgpr_64 = IMPLICIT_DEF
+ %3:sreg_32 = SI_IF %0, %bb.2, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
S_BRANCH %bb.1
bb.1:
- %12:vreg_64_align2 = IMPLICIT_DEF
- %12:vreg_64_align2 = IMPLICIT_DEF
- GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed %3, killed %12, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ successors: %bb.2
+
+ $m0 = S_MOV_B32 0
+ CLUSTER_LOAD_ASYNC_TO_LDS_B32_SADDR %1, %2, %1, 0, 0, implicit-def dead $asynccnt, implicit $m0, implicit $exec, implicit $asynccnt
+ $m0 = S_MOV_B32 1
bb.2:
- $exec_lo = S_OR_B32 $exec_lo, killed %0, implicit-def $scc
- S_ENDPGM 0
+ SI_END_CF %3, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
...
-# A post-DMA dependency on an operand prevents sinking.
---
-name: no_sink_post_dma_dependency
+name: no_sink_mismatched_end_cf
tracksRegLiveness: true
-isSSA: false
-machineFunctionInfo:
- isEntryFunction: true
- sgprForEXECCopy: '$sgpr105'
body: |
- ; CHECK-LABEL: name: no_sink_post_dma_dependency
+ ; CHECK-LABEL: name: no_sink_mismatched_end_cf
; CHECK: bb.0:
; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
- ; CHECK-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF3:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[SI_IF:%[0-9]+]]:sreg_32 = SI_IF [[DEF]], %bb.2, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 [[DEF]], killed [[DEF1]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY killed [[DEF]]
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128_SADDR [[DEF2]], [[DEF3]], [[DEF2]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: $exec_lo = S_OR_B32 $exec_lo, killed [[COPY]], implicit-def $scc
- ; CHECK-NEXT: S_ENDPGM 0
+ ; CHECK-NEXT: SI_END_CF [[DEF1]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
bb.0:
successors: %bb.1, %bb.2
- %3:vgpr_32 = IMPLICIT_DEF
- %12:vreg_64_align2 = IMPLICIT_DEF
- %0:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
- S_CBRANCH_EXECZ %bb.2, implicit $exec
+
+ %0:sreg_32 = IMPLICIT_DEF
+ %1:sreg_32 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ %3:sgpr_64 = IMPLICIT_DEF
+ %4:sreg_32 = SI_IF %0, %bb.2, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
S_BRANCH %bb.1
bb.1:
- GLOBAL_LOAD_ASYNC_TO_LDS_B128 %3, killed %12, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
- %20:vgpr_32 = COPY killed %3
+ successors: %bb.2
+
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128_SADDR %2, %3, %2, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
bb.2:
- $exec_lo = S_OR_B32 $exec_lo, killed %0, implicit-def $scc
- S_ENDPGM 0
+ SI_END_CF %1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
...
-# A third join predecessor is outside the EXECZ region, so the DMA must stay in
-# bb.2.
+
---
name: no_sink_extra_join_predecessor
tracksRegLiveness: true
-isSSA: false
-machineFunctionInfo:
- isEntryFunction: true
- sgprForEXECCopy: '$sgpr105'
body: |
; CHECK-LABEL: name: no_sink_extra_join_predecessor
; CHECK: bb.0:
; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CHECK-NEXT: [[DEF1:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
- ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
- ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.3, implicit undef $scc
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[SI_IF:%[0-9]+]]:sreg_32 = SI_IF [[DEF]], %bb.3, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.4(0x40000000)
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: S_CBRANCH_EXECZ %bb.4, implicit $exec
- ; CHECK-NEXT: S_BRANCH %bb.2
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128_SADDR [[DEF1]], [[DEF2]], [[DEF1]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: S_BRANCH %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: successors: %bb.4(0x80000000)
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR [[DEF1]], [[DEF2]], [[DEF]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
- ; CHECK-NEXT: S_BRANCH %bb.4
+ ; CHECK-NEXT: S_BRANCH %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: successors: %bb.4(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: S_BRANCH %bb.4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.4:
- ; CHECK-NEXT: $exec_lo = S_OR_B32 $exec_lo, [[COPY]], implicit-def $scc
- ; CHECK-NEXT: S_ENDPGM 0
+ ; CHECK-NEXT: SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
bb.0:
successors: %bb.1, %bb.3
- %3:vgpr_32 = IMPLICIT_DEF
- %6:sgpr_64 = IMPLICIT_DEF
- %9:vgpr_32 = IMPLICIT_DEF
- %0:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
- S_CBRANCH_SCC1 %bb.3, implicit undef $scc
+
+ %0:sreg_32 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ %2:sgpr_64 = IMPLICIT_DEF
+ %3:sreg_32 = SI_IF %0, %bb.3, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
S_BRANCH %bb.1
bb.1:
- successors: %bb.2, %bb.4
- S_CBRANCH_EXECZ %bb.4, implicit $exec
- S_BRANCH %bb.2
+ successors: %bb.3
+
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128_SADDR %1, %2, %1, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ S_BRANCH %bb.3
bb.2:
- successors: %bb.4
- GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR %6, %9, %3, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
- S_BRANCH %bb.4
+ successors: %bb.3
- bb.3:
- successors: %bb.4
- S_BRANCH %bb.4
+ S_BRANCH %bb.3
- bb.4:
- $exec_lo = S_OR_B32 $exec_lo, %0, implicit-def $scc
- S_ENDPGM 0
+ bb.3:
+ SI_END_CF %3, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
...
-# An arbitrary EXEC write is not an SI_END_CF restore.
---
-name: no_sink_wrong_exec_restore
+name: no_sink_join_not_successor_of_head
tracksRegLiveness: true
-isSSA: false
-machineFunctionInfo:
- isEntryFunction: true
- sgprForEXECCopy: '$sgpr105'
body: |
- ; CHECK-LABEL: name: no_sink_wrong_exec_restore
+ ; CHECK-LABEL: name: no_sink_join_not_successor_of_head
; CHECK: bb.0:
; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CHECK-NEXT: [[DEF1:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
- ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
- ; CHECK-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[SI_IF:%[0-9]+]]:sreg_32 = SI_IF [[DEF]], %bb.3, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR [[DEF1]], [[DEF2]], [[DEF]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: $exec_lo = S_MOV_B32 0
- ; CHECK-NEXT: S_ENDPGM 0
- bb.0:
- successors: %bb.1, %bb.2
- %3:vgpr_32 = IMPLICIT_DEF
- %6:sgpr_64 = IMPLICIT_DEF
- %9:vgpr_32 = IMPLICIT_DEF
- %0:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
- S_CBRANCH_EXECZ %bb.2, implicit $exec
- S_BRANCH %bb.1
-
- bb.1:
- GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR %6, %9, %3, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
-
- bb.2:
- $exec_lo = S_MOV_B32 0
- S_ENDPGM 0
-...
-
-# Multiple DMAs retain their order without treating a later DMA's local
-# operand definition as a crossed dependency.
----
-name: sink_multiple_dmas
-tracksRegLiveness: true
-isSSA: false
-machineFunctionInfo:
- isEntryFunction: true
- sgprForEXECCopy: '$sgpr105'
-body: |
- ; CHECK-LABEL: name: sink_multiple_dmas
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF1:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
- ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
- ; CHECK-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CHECK-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; CHECK-NEXT: S_BRANCH %bb.1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128_SADDR [[V_MOV_B32_e32_]], [[DEF1]], [[V_MOV_B32_e32_]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: S_BRANCH %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: liveins: $asynccnt, $exec
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR [[DEF1]], [[DEF2]], [[DEF]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
- ; CHECK-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR [[DEF1]], [[DEF3]], [[DEF]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
- ; CHECK-NEXT: $exec_lo = S_OR_B32 $exec_lo, [[COPY]], implicit-def $scc
- ; CHECK-NEXT: S_ENDPGM 0
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
bb.0:
successors: %bb.1, %bb.2
- %3:vgpr_32 = IMPLICIT_DEF
- %6:sgpr_64 = IMPLICIT_DEF
- %9:vgpr_32 = IMPLICIT_DEF
- %0:sreg_32 = COPY $exec_lo, implicit-def $exec_lo
- S_CBRANCH_EXECZ %bb.2, implicit $exec
+
+ %0:sreg_32 = IMPLICIT_DEF
+ %1:sreg_32 = SI_IF %0, %bb.3, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
S_BRANCH %bb.1
bb.1:
- GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR %6, %9, %3, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
- %11:vgpr_32 = V_MOV_B32_e32 4, implicit $exec
- GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR %6, %11, %3, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ successors: %bb.3
+
+ %2:sgpr_64 = IMPLICIT_DEF
+ %3:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128_SADDR %3, %2, %3, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ S_BRANCH %bb.3
bb.2:
- $exec_lo = S_OR_B32 $exec_lo, %0, implicit-def $scc
- S_ENDPGM 0
+ successors: %bb.3
+
+ S_BRANCH %bb.3
+
+ bb.3:
+ SI_END_CF %1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
...
diff --git a/llvm/test/CodeGen/AMDGPU/sink-async-dma-out-of-execz.ll b/llvm/test/CodeGen/AMDGPU/sink-async-dma-out-of-execz.ll
index 73ad89ead3b53..c88f7d01d3832 100644
--- a/llvm/test/CodeGen/AMDGPU/sink-async-dma-out-of-execz.ll
+++ b/llvm/test/CodeGen/AMDGPU/sink-async-dma-out-of-execz.ll
@@ -1,79 +1,178 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX1250 %s
+; RUN: llc -mtriple=amdgpu12.50 < %s | FileCheck %s -check-prefixes=GFX1250
-; The SISinkAsyncDMA pass sinks async DMA out of s_cbranch_execz-guarded
-; then-blocks into the join-block, immediately before the s_or_b32 EXEC
-; restore, so every wave issues the DMA and the ASYNCcnt observed at the join
-; point no longer depends on whether the wave skipped the block. This is a
-; determinism optimization; correctness never relies on it firing.
-
-
-; End-to-end: the async DMA sinks into %join right before the s_or_b32 EXEC
-; restore, and the block-local address def gets a header-side IMPLICIT_DEF.
-define amdgpu_ps void @async_store_simple(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr, i32 %bound) {
-; GFX1250-LABEL: async_store_simple:
+define amdgpu_ps void @sink_tightens_asyncmark_wait(ptr addrspace(1) inreg %src,
+; GFX1250-LABEL: sink_tightens_asyncmark_wait:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_mov_b32 s2, exec_lo
-; GFX1250-NEXT: v_cmpx_lt_i32_e32 s0, v1
+; GFX1250-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-NEXT: v_cmp_lt_i32_e32 vcc_lo, s0, v1
; GFX1250-NEXT: ; implicit-def: $vgpr1
-; GFX1250-NEXT: ; %bb.1: ; %do_store
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: global_load_async_to_lds_b32 v0, v2, s[0:1]
+; GFX1250-NEXT: ; asyncmark
+; GFX1250-NEXT: ; implicit-def: $vgpr2
+; GFX1250-NEXT: s_and_saveexec_b32 s2, vcc_lo
+; GFX1250-NEXT: ; %bb.1: ; %prefetch
+; GFX1250-NEXT: v_dual_mov_b32 v1, 0x100 :: v_dual_add_nc_u32 v2, 0x100, v0
; GFX1250-NEXT: ; %bb.2: ; %join
-; GFX1250-NEXT: global_store_async_from_lds_b128 v1, v0, s[0:1]
+; GFX1250-NEXT: global_load_async_to_lds_b32 v2, v1, s[0:1]
; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s2
-; GFX1250-NEXT: s_wait_asynccnt 0x0
+; GFX1250-NEXT: ; wait_asyncmark(0)
+; GFX1250-NEXT: s_wait_asynccnt 0x1
; GFX1250-NEXT: s_endpgm
+ ptr addrspace(3) %lds,
+ i32 %bound) {
entry:
+ tail call void @llvm.amdgcn.global.load.async.to.lds.b32(
+ ptr addrspace(1) %src, ptr addrspace(3) %lds, i32 0, i32 0)
+ tail call void @llvm.amdgcn.asyncmark()
%tid = tail call i32 @llvm.amdgcn.workitem.id.x()
%cmp = icmp slt i32 %tid, %bound
- br i1 %cmp, label %do_store, label %skip
+ br i1 %cmp, label %prefetch, label %join
-do_store:
- tail call void @llvm.amdgcn.global.store.async.from.lds.b128(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr, i32 0, i32 0)
+prefetch:
+ %src.next = getelementptr i32, ptr addrspace(1) %src, i32 64
+ %lds.next = getelementptr i32, ptr addrspace(3) %lds, i32 64
+ tail call void @llvm.amdgcn.global.load.async.to.lds.b32(
+ ptr addrspace(1) %src.next, ptr addrspace(3) %lds.next, i32 0, i32 0)
br label %join
-skip:
+join:
+ tail call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ ret void
+}
+
+define amdgpu_kernel void @sink_dma_operand_used_after(ptr addrspace(1) readonly %src,
+; GFX1250-LABEL: sink_dma_operand_used_after:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x3c nv
+; GFX1250-NEXT: v_and_b32_e32 v2, 0x3ff, v0
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GFX1250-NEXT: ; implicit-def: $vgpr3
+; GFX1250-NEXT: ; implicit-def: $vgpr4
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_cmp_gt_u32_e32 vcc_lo, s0, v2
+; GFX1250-NEXT: ; implicit-def: $sgpr0_sgpr1_sgpr2
+; GFX1250-NEXT: s_and_saveexec_b32 s3, vcc_lo
+; GFX1250-NEXT: s_cbranch_execz .LBB1_2
+; GFX1250-NEXT: ; %bb.1: ; %then
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_add_nc_u32 v4, s2, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_add_nc_u32_e32 v1, 4, v4
+; GFX1250-NEXT: .LBB1_2: ; %join
+; GFX1250-NEXT: global_load_async_to_lds_b128 v4, v3, s[0:1]
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(3) %lds,
+ ptr addrspace(1) %out,
+ i32 %bound) {
+entry:
+ %tid = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %active = icmp ult i32 %tid, %bound
+ br i1 %active, label %then, label %join
+
+then:
+ %lds.offset = getelementptr i8, ptr addrspace(3) %lds, i32 %tid
+ tail call void @llvm.amdgcn.global.load.async.to.lds.b128(
+ ptr addrspace(1) %src, ptr addrspace(3) %lds.offset, i32 0, i32 0)
+ %lds.int = ptrtoint ptr addrspace(3) %lds.offset to i32
+ %after = add i32 %lds.int, 4
br label %join
join:
- tail call void @llvm.amdgcn.s.wait.asynccnt(i16 0)
+ %result = phi i32 [ %after, %then ], [ 0, %entry ]
+ store i32 %result, ptr addrspace(1) %out
ret void
}
-; Negative test: an explicit s_wait_asynccnt inside the then-block disqualifies
-; sinking. Skipping the wait on the EXECZ path would observe a different
-; ASYNCcnt value, so the eligibility scan must bail and the branch must stay.
-define amdgpu_ps void @async_load_then_block_has_waitcnt(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr, i32 %bound) {
-; GFX1250-LABEL: async_load_then_block_has_waitcnt:
+define amdgpu_ps void @no_sink_asyncmark_in_then_block(ptr addrspace(1) inreg %src,
+; GFX1250-LABEL: no_sink_asyncmark_in_then_block:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s2, exec_lo
; GFX1250-NEXT: v_cmpx_lt_i32_e32 s0, v1
-; GFX1250-NEXT: s_cbranch_execz .LBB1_2
-; GFX1250-NEXT: ; %bb.1: ; %do_load
+; GFX1250-NEXT: s_cbranch_execz .LBB2_2
+; GFX1250-NEXT: ; %bb.1: ; %then
; GFX1250-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NEXT: global_load_async_to_lds_b128 v0, v1, s[0:1]
+; GFX1250-NEXT: global_load_async_to_lds_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: ; asyncmark
+; GFX1250-NEXT: ; wait_asyncmark(0)
; GFX1250-NEXT: s_wait_asynccnt 0x0
-; GFX1250-NEXT: .LBB1_2: ; %join
+; GFX1250-NEXT: .LBB2_2: ; %join
; GFX1250-NEXT: s_endpgm
+ ptr addrspace(3) %lds,
+ i32 %bound) {
entry:
%tid = tail call i32 @llvm.amdgcn.workitem.id.x()
%cmp = icmp slt i32 %tid, %bound
- br i1 %cmp, label %do_load, label %skip
+ br i1 %cmp, label %then, label %join
-do_load:
- tail call void @llvm.amdgcn.global.load.async.to.lds.b128(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr, i32 0, i32 0)
- tail call void @llvm.amdgcn.s.wait.asynccnt(i16 0)
+then:
+ tail call void @llvm.amdgcn.global.load.async.to.lds.b32(
+ ptr addrspace(1) %src, ptr addrspace(3) %lds, i32 0, i32 0)
+ tail call void @llvm.amdgcn.asyncmark()
+ tail call void @llvm.amdgcn.wait.asyncmark(i16 0)
br label %join
-skip:
+join:
+ ret void
+}
+
+define amdgpu_ps void @no_sink_setreg_after_dma(ptr addrspace(1) inreg %src,
+; GFX1250-LABEL: no_sink_setreg_after_dma:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: global_load_async_to_lds_b32 v0, v2, s[0:1]
+; GFX1250-NEXT: ; asyncmark
+; GFX1250-NEXT: v_cmpx_lt_i32_e32 s0, v1
+; GFX1250-NEXT: s_cbranch_execz .LBB3_2
+; GFX1250-NEXT: ; %bb.1: ; %prefetch
+; GFX1250-NEXT: v_dual_mov_b32 v1, 0x100 :: v_dual_add_nc_u32 v0, 0x100, v0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 0, 2), 3
+; GFX1250-NEXT: global_load_async_to_lds_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: .LBB3_2: ; %join
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX1250-NEXT: ; wait_asyncmark(0)
+; GFX1250-NEXT: s_wait_asynccnt 0x0
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(3) %lds,
+ i32 %bound) {
+entry:
+ tail call void @llvm.amdgcn.global.load.async.to.lds.b32(
+ ptr addrspace(1) %src, ptr addrspace(3) %lds, i32 0, i32 0)
+ tail call void @llvm.amdgcn.asyncmark()
+ %tid = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %cmp = icmp slt i32 %tid, %bound
+ br i1 %cmp, label %prefetch, label %join
+
+prefetch:
+ %src.next = getelementptr i32, ptr addrspace(1) %src, i32 64
+ %lds.next = getelementptr i32, ptr addrspace(3) %lds, i32 64
+ tail call void @llvm.amdgcn.global.load.async.to.lds.b32(
+ ptr addrspace(1) %src.next, ptr addrspace(3) %lds.next, i32 0, i32 0)
+ tail call void @llvm.amdgcn.s.setreg(i32 2049, i32 3)
br label %join
join:
+ tail call void @llvm.amdgcn.wait.asyncmark(i16 0)
ret void
}
+
+declare void @llvm.amdgcn.s.setreg(i32 immarg, i32)
More information about the llvm-commits
mailing list