[llvm] [AMDGPU] Model waterfall loop EXEC update as a terminator (PR #219519)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Sun Sep 13 12:03:19 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/219519
>From 027c184d365e8b7ecf7a79d96380587eebf51a9b Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Fri, 28 Aug 2026 18:12:22 +0200
Subject: [PATCH 1/3] [AMDGPU] Model waterfall loop EXEC update as a terminator
Defining a register normally disqualifies an instruction from being a terminator, so mark it unspillable instead
Skip this at -O0, since RegAllocFast ignores that and spills right after the def
---
llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h | 3 +
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 22 ++-
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 6 +
llvm/lib/Target/AMDGPU/SIInstructions.td | 2 +
.../Target/AMDGPU/SIOptimizeExecMasking.cpp | 6 +
.../buffer-fat-pointer-atomicrmw-fadd.ll | 8 +-
.../buffer-fat-pointer-atomicrmw-fmax.ll | 8 +-
.../buffer-fat-pointer-atomicrmw-fmin.ll | 8 +-
...e92561-restore-undef-scc-verifier-error.ll | 2 +-
.../legalize-amdgcn.raw.ptr.buffer.load.ll | 72 ++++-----
.../AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll | 2 +-
.../AMDGPU/llvm.amdgcn.dual_intersect_ray.ll | 2 +-
...mdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll | 6 +-
....amdgcn.struct.buffer.load.format.v3f16.ll | 6 +-
...cn.struct.ptr.buffer.atomic.fadd.v2bf16.ll | 4 +-
...gcn.struct.ptr.buffer.atomic.fadd_nortn.ll | 12 +-
...mdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll | 12 +-
...mdgcn.struct.ptr.buffer.atomic.fmax.f32.ll | 20 +--
...mdgcn.struct.ptr.buffer.atomic.fmin.f32.ll | 20 +--
...gcn.struct.ptr.buffer.load.format.v3f16.ll | 4 +-
llvm/test/CodeGen/AMDGPU/mixed-vmem-types.ll | 4 +-
.../AMDGPU/move-to-valu-vimage-vsample.ll | 12 +-
...uf-legalize-operands-non-ptr-intrinsics.ll | 80 +++++-----
.../CodeGen/AMDGPU/mubuf-legalize-operands.ll | 80 +++++-----
.../AMDGPU/mubuf-legalize-operands.mir | 16 +-
...r-descriptor-waterfall-loop-idom-update.ll | 4 +-
llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll | 12 +-
llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll | 8 +-
.../waterfall-loop-wrexec-terminator.mir | 150 ++++++++++++++++++
29 files changed, 386 insertions(+), 205 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/waterfall-loop-wrexec-terminator.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h b/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
index af14066c1bd46..fc1c005f25426 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
@@ -28,6 +28,7 @@ class LaneMaskConstants {
const unsigned AndN2SaveExecOpc;
const unsigned AndN2TermOpc;
const unsigned AndN2WrExecOpc; // GFX10+ (HasNoSdstCMPX) only
+ const unsigned AndN2WrExecTermOpc;
const unsigned AndSaveExecOpc;
const unsigned AndSaveExecTermOpc;
const unsigned BfmOpc;
@@ -55,6 +56,8 @@ class LaneMaskConstants {
: AMDGPU::S_ANDN2_B64_term),
AndN2WrExecOpc(IsWave32 ? AMDGPU::S_ANDN2_WREXEC_B32
: AMDGPU::S_ANDN2_WREXEC_B64),
+ AndN2WrExecTermOpc(IsWave32 ? AMDGPU::S_ANDN2_WREXEC_B32_term
+ : AMDGPU::S_ANDN2_WREXEC_B64_term),
AndSaveExecOpc(IsWave32 ? AMDGPU::S_AND_SAVEEXEC_B32
: AMDGPU::S_AND_SAVEEXEC_B64),
AndSaveExecTermOpc(IsWave32 ? AMDGPU::S_AND_SAVEEXEC_B32_term
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index c70080cd1551f..ff5ce9314576d 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -2104,6 +2104,13 @@ bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
MI.setDesc(get(AMDGPU::V_CMPX_EQ_U64_nosdst_e32));
break;
+ case AMDGPU::S_ANDN2_WREXEC_B64_term:
+ MI.setDesc(get(AMDGPU::S_ANDN2_WREXEC_B64));
+ break;
+ case AMDGPU::S_ANDN2_WREXEC_B32_term:
+ MI.setDesc(get(AMDGPU::S_ANDN2_WREXEC_B32));
+ break;
+
case AMDGPU::SI_SPILL_S32_TO_VGPR:
MI.setDesc(get(AMDGPU::V_WRITELANE_B32));
break;
@@ -3255,6 +3262,8 @@ bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
case AMDGPU::S_AND_SAVEEXEC_B32_term:
case AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term:
case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
+ case AMDGPU::S_ANDN2_WREXEC_B32_term:
+ case AMDGPU::S_ANDN2_WREXEC_B64_term:
break;
case AMDGPU::SI_IF:
case AMDGPU::SI_ELSE:
@@ -7475,9 +7484,8 @@ static void emitLoadScalarOpsFromVGPRLoop(
}
}
- // Instructions AndSaveExecOpc and AndN2WrExecOpc that modify EXEC mask
- // should have isTerminator=1 but terminators that define
- // virtual registers are not supported.
+ // AndSaveExecOpc modifies EXEC but can't be isTerminator=1: terminators
+ // that define virtual registers aren't supported.
Register SaveExec;
if (!UseNewExecInstructions) {
SaveExec = MRI.createVirtualRegister(BoolXExecRC);
@@ -7492,8 +7500,14 @@ static void emitLoadScalarOpsFromVGPRLoop(
I = BodyBB.end();
if (UseNewExecInstructions) {
+ // Terminator form lets PHI elimination fold this into the exec PHI's
+ // def. Skip it at -O0: RegAllocFast spills live-out defs right after
+ // them, not at the first terminator.
+ bool UseTermForm = MF.getTarget().getOptLevel() != CodeGenOptLevel::None;
MRI.setSimpleHint(NewExec, PhiExec);
- BuildMI(BodyBB, I, DL, TII.get(LMC.AndN2WrExecOpc), NewExec)
+ BuildMI(BodyBB, I, DL,
+ TII.get(UseTermForm ? LMC.AndN2WrExecTermOpc : LMC.AndN2WrExecOpc),
+ NewExec)
.addReg(PhiExec);
} else {
// Update EXEC, switch all done bits to 0 and all todo bits to 1.
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 4bf0f557ae4e1..98381005b3c86 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -351,6 +351,12 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
bool expandPostRAPseudo(MachineInstr &MI) const override;
+ // See the comment in emitLoadScalarOpsFromVGPRLoop.
+ bool isUnspillableTerminatorImpl(const MachineInstr *MI) const override {
+ return MI->getOpcode() == AMDGPU::S_ANDN2_WREXEC_B32_term ||
+ MI->getOpcode() == AMDGPU::S_ANDN2_WREXEC_B64_term;
+ }
+
void
reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
Register DestReg, unsigned SubIdx, const MachineInstr &Orig,
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index eb23b6fea26b3..a45d424a6a882 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -497,6 +497,7 @@ def S_OR_B64_term : WrapTerminatorInst<S_OR_B64>;
def S_ANDN2_B64_term : WrapTerminatorInst<S_ANDN2_B64>;
def S_AND_B64_term : WrapTerminatorInst<S_AND_B64>;
def S_AND_SAVEEXEC_B64_term : WrapTerminatorInst<S_AND_SAVEEXEC_B64>;
+def S_ANDN2_WREXEC_B64_term : WrapTerminatorInst<S_ANDN2_WREXEC_B64>;
}
let WaveSizePredicate = isWave32 in {
@@ -506,6 +507,7 @@ def S_OR_B32_term : WrapTerminatorInst<S_OR_B32>;
def S_ANDN2_B32_term : WrapTerminatorInst<S_ANDN2_B32>;
def S_AND_B32_term : WrapTerminatorInst<S_AND_B32>;
def S_AND_SAVEEXEC_B32_term : WrapTerminatorInst<S_AND_SAVEEXEC_B32>;
+def S_ANDN2_WREXEC_B32_term : WrapTerminatorInst<S_ANDN2_WREXEC_B32>;
}
class WrapTerminatorVOPC<VOPC_Pseudo base_inst> : VPseudoInstSI<
diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
index a9bd72e634414..fab31365eca17 100644
--- a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
+++ b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
@@ -299,6 +299,12 @@ bool SIOptimizeExecMasking::removeTerminatorBit(MachineInstr &MI) const {
case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
MI.setDesc(TII->get(AMDGPU::V_CMPX_EQ_U64_nosdst_e32));
return true;
+ case AMDGPU::S_ANDN2_WREXEC_B64_term:
+ MI.setDesc(TII->get(AMDGPU::S_ANDN2_WREXEC_B64));
+ return true;
+ case AMDGPU::S_ANDN2_WREXEC_B32_term:
+ MI.setDesc(TII->get(AMDGPU::S_ANDN2_WREXEC_B32));
+ return true;
default:
return false;
}
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index 4719d6edbaaa5..b2cf394ccf193 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -387,9 +387,9 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_add_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: s_cbranch_execnz .LBB2_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
@@ -444,9 +444,9 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_add_f32 v5, v4, s[0:3], 0 offen offset:1024 glc
-; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-NEXT: ; implicit-def: $vgpr4
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: s_cbranch_execnz .LBB2_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s4
@@ -7036,9 +7036,9 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_pk_add_f16 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
@@ -9857,9 +9857,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_pk_add_bf16 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: s_cbranch_execnz .LBB28_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index cecc14be3c521..3fdd9467a74f2 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -378,9 +378,9 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_max_num_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: s_cbranch_execnz .LBB2_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
@@ -466,9 +466,9 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_max_f32 v5, v4, s[0:3], 0 offen offset:1024 glc
-; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-NEXT: ; implicit-def: $vgpr4
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: s_cbranch_execnz .LBB2_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s4
@@ -494,9 +494,9 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmax v5, v4, s[4:7], 0 offen offset:1024 glc
-; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB2_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -1710,9 +1710,9 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmax_x2 v[5:6], v4, s[4:7], 0 offen offset:2048 glc
-; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB7_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index 7859ac4841a10..c6cd077701cdd 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -378,9 +378,9 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_min_num_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: s_cbranch_execnz .LBB2_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
@@ -466,9 +466,9 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_min_f32 v5, v4, s[0:3], 0 offen offset:1024 glc
-; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-NEXT: ; implicit-def: $vgpr4
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: s_cbranch_execnz .LBB2_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s4
@@ -494,9 +494,9 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmin v5, v4, s[4:7], 0 offen offset:1024 glc
-; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB2_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -1710,9 +1710,9 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmin_x2 v[5:6], v4, s[4:7], 0 offen offset:2048 glc
-; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB7_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
diff --git a/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll b/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
index 124ba142a56f6..e22aeee59af7a 100644
--- a/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
+++ b/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
@@ -37,8 +37,8 @@ define void @issue92561(ptr addrspace(1) %arg) {
; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4)
; SDAG-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; SDAG-NEXT: image_sample_c_lz v9, [v8, v8, v8, v8], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
-; SDAG-NEXT: s_and_not1_wrexec_b32 s13, s13
; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; SDAG-NEXT: s_and_not1_wrexec_b32 s13, s13
; SDAG-NEXT: s_cbranch_execnz .LBB0_1
; SDAG-NEXT: ; %bb.2:
; SDAG-NEXT: s_mov_b32 exec_lo, s12
diff --git a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll
index 76d19d9c6a986..3c7394d9753ca 100644
--- a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll
@@ -95,7 +95,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(ptr
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -199,7 +199,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__sgpr_voffset__sgpr_soffset(ptr
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -303,7 +303,7 @@ define float @raw_ptr_buffer_load_f32__vgpr_rsrc__vgpr_voffset__sgpr_soffset(ptr
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -407,7 +407,7 @@ define float @raw_ptr_buffer_load_f32__vgpr_rsrc__vgpr_voffset__vgpr_soffset(ptr
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -511,7 +511,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 1, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -615,7 +615,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 2, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -719,7 +719,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 4, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -823,7 +823,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 6, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -927,7 +927,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 5, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -1031,7 +1031,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 7, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -1138,7 +1138,7 @@ define <2 x float> @raw_ptr_buffer_load_v2f32__sgpr_rsrc__vgpr_voffset__sgpr_sof
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s64) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -1249,7 +1249,7 @@ define <3 x float> @raw_ptr_buffer_load_v3f32__sgpr_rsrc__vgpr_voffset__sgpr_sof
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN:%[0-9]+]]:vreg_96_align2 = BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s96) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -1364,7 +1364,7 @@ define <4 x float> @raw_ptr_buffer_load_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_sof
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -1475,7 +1475,7 @@ define half @raw_ptr_buffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(ptr
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -1579,7 +1579,7 @@ define <2 x half> @raw_ptr_buffer_load_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -1685,7 +1685,7 @@ define <4 x half> @raw_ptr_buffer_load_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s64) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -1791,7 +1791,7 @@ define float @raw_ptr_buffer_load_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffset_zext
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_UBYTE_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8) from %ir.rsrc, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -1896,7 +1896,7 @@ define float @raw_ptr_buffer_load_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sext
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_SBYTE_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SBYTE_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8) from %ir.rsrc, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -2001,7 +2001,7 @@ define float @raw_ptr_buffer_load_i16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_zex
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -2106,7 +2106,7 @@ define float @raw_ptr_buffer_load_i16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sex
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_SSHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SSHORT_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -2213,7 +2213,7 @@ define half @raw_ptr_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset(ptr
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -2318,7 +2318,7 @@ define float @raw_ptr_buffer_load_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffset(ptr
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_UBYTE_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8) from %ir.rsrc, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -2421,7 +2421,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vdpr_voffset__sgpr_soffset__vo
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -2522,7 +2522,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__vo
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -2624,7 +2624,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__vo
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 4096, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -2721,7 +2721,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset0(ptr a
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE5]], $sgpr_null, 8388604, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -2826,7 +2826,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_vof
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -2932,7 +2932,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__vo
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -3040,7 +3040,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__vo
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -3139,7 +3139,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sof
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY]], killed [[REG_SEQUENCE5]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -3237,7 +3237,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sof
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY]], killed [[REG_SEQUENCE5]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -3344,7 +3344,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sof
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -3452,7 +3452,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sof
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -3560,7 +3560,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sof
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -3669,7 +3669,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sof
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -3778,7 +3778,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_vof
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll
index 7aa4816d26413..f686e13303d49 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll
@@ -180,11 +180,11 @@ define amdgpu_ps <10 x float> @image_bvh8_intersect_ray_vvvvvv(i64 %node_ptr, fl
; GFX12-SDAG-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[12:13]
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: image_bvh8_intersect_ray v[0:9], [v[24:25], v[26:27], v[21:23], v[18:20], v28], s[0:3]
-; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr10_vgpr11_vgpr12_vgpr13
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr24_vgpr25
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr26_vgpr27
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr28
+; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-SDAG-NEXT: s_cbranch_execnz .LBB3_1
; GFX12-SDAG-NEXT: ; %bb.2:
; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll
index b843d8394f228..a07960f803250 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll
@@ -185,11 +185,11 @@ define amdgpu_ps <10 x float> @image_bvh_dual_intersect_ray_vvvvvv(i64 %node_ptr
; GFX12-SDAG-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[13:14]
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: image_bvh_dual_intersect_ray v[0:9], [v[27:28], v[29:30], v[22:24], v[19:21], v[25:26]], s[0:3]
-; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr11_vgpr12_vgpr13_vgpr14
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr27_vgpr28
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr29_vgpr30
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr25_vgpr26
+; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-SDAG-NEXT: s_cbranch_execnz .LBB3_1
; GFX12-SDAG-NEXT: ; %bb.2:
; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
index de35c48f59e03..82c3874fe6b0d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
@@ -120,10 +120,10 @@ define <2 x bfloat> @raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v6
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v5, s[0:3], s6 offen offset:128 th:TH_ATOMIC_RETURN
-; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr6
; GFX1200-NEXT: ; implicit-def: $vgpr5
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: s_cbranch_execnz .LBB4_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -154,11 +154,11 @@ define <2 x bfloat> @raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__
; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v6
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], s6 offen th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9_vgpr10_vgpr11
; GFX1250-NEXT: ; implicit-def: $vgpr6
; GFX1250-NEXT: ; implicit-def: $vgpr1
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: s_cbranch_execnz .LBB4_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
index 574b45472f9f1..c65422bc86810 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
@@ -23,9 +23,9 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX10-NEXT: buffer_load_format_d16_xyz v[5:6], v4, s[0:3], 0 idxen
-; GFX10-NEXT: s_andn2_wrexec_b32 s5, s5
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
+; GFX10-NEXT: s_andn2_wrexec_b32 s5, s5
; GFX10-NEXT: s_cbranch_execnz .LBB0_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -104,9 +104,9 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
-; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-NEXT: ; implicit-def: $vgpr4
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s4
@@ -131,9 +131,9 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], null idxen
-; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: s_cbranch_execnz .LBB0_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
index 1dca37a2f8fcc..6c8756b7e5f2f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
@@ -56,10 +56,10 @@ define <2 x bfloat> @struct_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsr
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v[5:6], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
-; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: s_cbranch_execnz .LBB2_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -93,11 +93,11 @@ define void @struct_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__vgpr_rsrc__vgp
; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v[5:6], s[0:3], s6 idxen offen
-; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr0
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: s_cbranch_execnz .LBB3_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
index 2a631b543f5aa..e6517afddc410 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
@@ -311,11 +311,11 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: buffer_atomic_add_f32 v0, v[5:6], s[0:3], s6 idxen offen
-; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr0
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: s_cbranch_execnz .LBB4_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -345,12 +345,12 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[0:3], s6 idxen offen
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr0
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: s_cbranch_execnz .LBB4_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s4
@@ -481,11 +481,11 @@ define void @struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr
; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: buffer_atomic_pk_add_f16 v0, v[5:6], s[0:3], s6 idxen offen
-; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr0
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: s_cbranch_execnz .LBB5_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -515,12 +515,12 @@ define void @struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[0:3], s6 idxen offen
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr0
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: s_cbranch_execnz .LBB5_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
index c3e88997d0ba3..729b2126a88e5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
@@ -261,10 +261,10 @@ define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_add_f32 v0, v[5:6], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
-; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: s_cbranch_execnz .LBB4_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -296,11 +296,11 @@ define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: s_cbranch_execnz .LBB4_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s4
@@ -403,10 +403,10 @@ define <2 x half> @struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_pk_add_f16 v0, v[5:6], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
-; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: s_cbranch_execnz .LBB5_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -438,11 +438,11 @@ define <2 x half> @struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__
; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: s_cbranch_execnz .LBB5_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
index f4d364eaa4458..b40d866802f9b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
@@ -473,9 +473,9 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmax v0, v[5:6], s[4:7], s16 idxen offen offset:256 glc
-; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX10-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB8_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -499,9 +499,9 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_max_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 glc
-; GFX11-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX11-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX11-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX11-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX11-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s1
@@ -529,9 +529,9 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_max_num_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 th:TH_ATOMIC_RETURN
-; GFX1200-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX1200-NEXT: s_cbranch_execnz .LBB8_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s1
@@ -560,10 +560,10 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[4:5]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_max_num_f32 v0, v[8:9], s[4:7], s0 idxen offen th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX1250-NEXT: s_cbranch_execnz .LBB8_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s1
@@ -649,10 +649,10 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX10-NEXT: v_cmpx_eq_u32_e32 s10, v7
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmax v0, v[5:6], s[4:7], s10 idxen offen offset:256 glc
-; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX10-NEXT: ; implicit-def: $vgpr7
; GFX10-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB9_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -679,10 +679,10 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX11-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_max_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 glc
-; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX11-NEXT: ; implicit-def: $vgpr7
; GFX11-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s4
@@ -712,10 +712,10 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_max_num_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 th:TH_ATOMIC_RETURN
-; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: s_cbranch_execnz .LBB9_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -746,11 +746,11 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_max_num_f32 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: s_cbranch_execnz .LBB9_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
index d284c476c9333..edabefdfdaf8d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
@@ -473,9 +473,9 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmin v0, v[5:6], s[4:7], s16 idxen offen offset:256 glc
-; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX10-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB8_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -499,9 +499,9 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_min_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 glc
-; GFX11-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX11-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX11-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX11-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX11-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s1
@@ -529,9 +529,9 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_min_num_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 th:TH_ATOMIC_RETURN
-; GFX1200-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX1200-NEXT: s_cbranch_execnz .LBB8_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s1
@@ -560,10 +560,10 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[4:5]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_min_num_f32 v0, v[8:9], s[4:7], s0 idxen offen th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX1250-NEXT: s_cbranch_execnz .LBB8_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s1
@@ -649,10 +649,10 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX10-NEXT: v_cmpx_eq_u32_e32 s10, v7
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmin v0, v[5:6], s[4:7], s10 idxen offen offset:256 glc
-; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX10-NEXT: ; implicit-def: $vgpr7
; GFX10-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB9_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -679,10 +679,10 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX11-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_min_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 glc
-; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX11-NEXT: ; implicit-def: $vgpr7
; GFX11-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s4
@@ -712,10 +712,10 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_min_num_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 th:TH_ATOMIC_RETURN
-; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: s_cbranch_execnz .LBB9_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -746,11 +746,11 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_min_num_f32 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: s_cbranch_execnz .LBB9_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
index 1db4db27e8841..e0752819e7145 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
@@ -20,9 +20,9 @@ define amdgpu_gs void @main(ptr addrspace(8) %arg, i32 %arg1) {
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX10-NEXT: buffer_load_format_d16_xyz v[5:6], v4, s[0:3], 0 idxen
-; GFX10-NEXT: s_andn2_wrexec_b32 s5, s5
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
+; GFX10-NEXT: s_andn2_wrexec_b32 s5, s5
; GFX10-NEXT: s_cbranch_execnz .LBB0_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -101,9 +101,9 @@ define amdgpu_gs void @main(ptr addrspace(8) %arg, i32 %arg1) {
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
-; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-NEXT: ; implicit-def: $vgpr4
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/mixed-vmem-types.ll b/llvm/test/CodeGen/AMDGPU/mixed-vmem-types.ll
index 7fc67b832a9d8..15ffb3a7e7dcf 100644
--- a/llvm/test/CodeGen/AMDGPU/mixed-vmem-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/mixed-vmem-types.ll
@@ -200,13 +200,13 @@ define amdgpu_cs void @mixed_vmem_types_bvh_reads(i32 inreg %descTable0, i32 inr
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[16:17], v[12:13]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[18:19], v[14:15]
; GFX11-NEXT: image_bvh64_intersect_ray v[15:18], v[0:11], s[16:19]
-; GFX11-NEXT: s_and_not1_wrexec_b32 s21, s21
; GFX11-NEXT: ; implicit-def: $vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX11-NEXT: ; implicit-def: $vgpr2
; GFX11-NEXT: ; implicit-def: $vgpr3_vgpr4_vgpr5
; GFX11-NEXT: ; implicit-def: $vgpr6_vgpr7_vgpr8
; GFX11-NEXT: ; implicit-def: $vgpr9_vgpr10_vgpr11
+; GFX11-NEXT: s_and_not1_wrexec_b32 s21, s21
; GFX11-NEXT: s_cbranch_execnz .LBB1_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s20
@@ -257,13 +257,13 @@ define amdgpu_cs void @mixed_vmem_types_bvh_reads(i32 inreg %descTable0, i32 inr
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[18:19], v[14:15]
; GFX12-NEXT: image_bvh64_intersect_ray v[15:18], [v[0:1], v2, v[3:5], v[6:8], v[9:11]], s[16:19]
-; GFX12-NEXT: s_and_not1_wrexec_b32 s21, s21
; GFX12-NEXT: ; implicit-def: $vgpr12_vgpr13_vgpr14_vgpr15
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT: ; implicit-def: $vgpr2
; GFX12-NEXT: ; implicit-def: $vgpr3_vgpr4_vgpr5
; GFX12-NEXT: ; implicit-def: $vgpr6_vgpr7_vgpr8
; GFX12-NEXT: ; implicit-def: $vgpr9_vgpr10_vgpr11
+; GFX12-NEXT: s_and_not1_wrexec_b32 s21, s21
; GFX12-NEXT: s_cbranch_execnz .LBB1_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s20
diff --git a/llvm/test/CodeGen/AMDGPU/move-to-valu-vimage-vsample.ll b/llvm/test/CodeGen/AMDGPU/move-to-valu-vimage-vsample.ll
index ed8fef40547d9..289333f1d41e7 100644
--- a/llvm/test/CodeGen/AMDGPU/move-to-valu-vimage-vsample.ll
+++ b/llvm/test/CodeGen/AMDGPU/move-to-valu-vimage-vsample.ll
@@ -47,7 +47,7 @@ define amdgpu_ps float @vimage_move_to_valu(<8 x i32> %rsrc) {
; GFX11-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[IMAGE_LOAD_V1_V2_nsa_gfx11_:%[0-9]+]]:vgpr_32 = IMAGE_LOAD_V1_V2_nsa_gfx11 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE5]], 1, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX11-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX11-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.3:
@@ -99,7 +99,7 @@ define amdgpu_ps float @vimage_move_to_valu(<8 x i32> %rsrc) {
; GFX12-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[IMAGE_LOAD_V1_V2_gfx12_:%[0-9]+]]:vgpr_32 = IMAGE_LOAD_V1_V2_gfx12 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE5]], 1, 1, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX12-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX12-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
@@ -161,7 +161,7 @@ define amdgpu_ps float @vsample_move_to_valu_rsrc(<8 x i32> %rsrc, <4 x i32> inr
; GFX11-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[IMAGE_SAMPLE_V1_V1_gfx11_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V1_gfx11 [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE6]], [[REG_SEQUENCE1]], 1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX11-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX11-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.3:
@@ -218,7 +218,7 @@ define amdgpu_ps float @vsample_move_to_valu_rsrc(<8 x i32> %rsrc, <4 x i32> inr
; GFX12-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[IMAGE_SAMPLE_V1_V1_gfx12_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V1_gfx12 [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE6]], [[REG_SEQUENCE1]], 1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX12-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX12-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
@@ -272,7 +272,7 @@ define amdgpu_ps float @vsample_move_to_valu_samp(<8 x i32> inreg %rsrc, <4 x i3
; GFX11-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[IMAGE_SAMPLE_V1_V1_gfx11_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V1_gfx11 [[V_MOV_B32_e32_]], [[REG_SEQUENCE]], killed [[REG_SEQUENCE4]], 1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX11-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX11-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.3:
@@ -321,7 +321,7 @@ define amdgpu_ps float @vsample_move_to_valu_samp(<8 x i32> inreg %rsrc, <4 x i3
; GFX12-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[IMAGE_SAMPLE_V1_V1_gfx12_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V1_gfx12 [[V_MOV_B32_e32_]], [[REG_SEQUENCE]], killed [[REG_SEQUENCE4]], 1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX12-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX12-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
index 610d80e9bba09..7163c4607b8b4 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
@@ -49,9 +49,9 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W32-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
-; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB0_1
; GFX1010_W32-NEXT: ; %bb.2:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -74,9 +74,9 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W64-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
-; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB0_1
; GFX1010_W64-NEXT: ; %bb.2:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -100,9 +100,9 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W32-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
-; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB0_1
; GFX1100_W32-NEXT: ; %bb.2:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
@@ -125,9 +125,9 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W64-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
-; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB0_1
; GFX1100_W64-NEXT: ; %bb.2:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
@@ -302,8 +302,8 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W32-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
-; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB1_1
; GFX1010_W32-NEXT: ; %bb.2:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -319,9 +319,9 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; GFX1010_W32-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
-; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W32-NEXT: ; implicit-def: $vgpr8
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB1_3
; GFX1010_W32-NEXT: ; %bb.4:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -348,8 +348,8 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W64-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
-; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB1_1
; GFX1010_W64-NEXT: ; %bb.2:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -365,9 +365,9 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; GFX1010_W64-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
-; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W64-NEXT: ; implicit-def: $vgpr8
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB1_3
; GFX1010_W64-NEXT: ; %bb.4:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -395,8 +395,8 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W32-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
-; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB1_1
; GFX1100_W32-NEXT: ; %bb.2:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
@@ -412,9 +412,9 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
; GFX1100_W32-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
-; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W32-NEXT: ; implicit-def: $vgpr8
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB1_3
; GFX1100_W32-NEXT: ; %bb.4:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
@@ -441,8 +441,8 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W64-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
-; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB1_1
; GFX1100_W64-NEXT: ; %bb.2:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
@@ -458,9 +458,9 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
; GFX1100_W64-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
-; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W64-NEXT: ; implicit-def: $vgpr8
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB1_3
; GFX1100_W64-NEXT: ; %bb.4:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
@@ -759,7 +759,7 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W32-NEXT: ;;#ASMSTART
; GFX1010_W32-NEXT: s_mov_b32 s4, 17
; GFX1010_W32-NEXT: ;;#ASMEND
-; GFX1010_W32-NEXT: v_mov_b32_e32 v9, s4
+; GFX1010_W32-NEXT: v_mov_b32_e32 v8, s4
; GFX1010_W32-NEXT: s_mov_b32 s5, exec_lo
; GFX1010_W32-NEXT: s_mov_b32 s6, s5
; GFX1010_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
@@ -770,10 +770,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[8:9], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[10:11], v[2:3]
-; GFX1010_W32-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
-; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s6, s6
+; GFX1010_W32-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr9
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr8
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s6, s6
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB2_1
; GFX1010_W32-NEXT: ; %bb.2:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -794,10 +794,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
-; GFX1010_W32-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
-; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX1010_W32-NEXT: buffer_load_format_x v9, v0, s[4:7], 0 idxen
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB2_4
; GFX1010_W32-NEXT: ; %bb.5:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -805,7 +805,7 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W32-NEXT: .LBB2_6: ; %bb2
; GFX1010_W32-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W32-NEXT: global_store_dword v[11:12], v8, off
+; GFX1010_W32-NEXT: global_store_dword v[11:12], v9, off
; GFX1010_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W32-NEXT: s_setpc_b64 s[30:31]
;
@@ -815,7 +815,7 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W64-NEXT: ;;#ASMSTART
; GFX1010_W64-NEXT: s_mov_b32 s4, 17
; GFX1010_W64-NEXT: ;;#ASMEND
-; GFX1010_W64-NEXT: v_mov_b32_e32 v9, s4
+; GFX1010_W64-NEXT: v_mov_b32_e32 v8, s4
; GFX1010_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX1010_W64-NEXT: s_mov_b64 s[12:13], s[6:7]
; GFX1010_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
@@ -826,10 +826,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[8:9], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[10:11], v[2:3]
-; GFX1010_W64-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
-; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
+; GFX1010_W64-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr9
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr8
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB2_1
; GFX1010_W64-NEXT: ; %bb.2:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -850,10 +850,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
-; GFX1010_W64-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
-; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
+; GFX1010_W64-NEXT: buffer_load_format_x v9, v0, s[4:7], 0 idxen
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB2_4
; GFX1010_W64-NEXT: ; %bb.5:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -861,7 +861,7 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W64-NEXT: .LBB2_6: ; %bb2
; GFX1010_W64-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W64-NEXT: global_store_dword v[11:12], v8, off
+; GFX1010_W64-NEXT: global_store_dword v[11:12], v9, off
; GFX1010_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W64-NEXT: s_setpc_b64 s[30:31]
;
@@ -871,7 +871,7 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W32-NEXT: ;;#ASMSTART
; GFX1100_W32-NEXT: s_mov_b32 s4, 17
; GFX1100_W32-NEXT: ;;#ASMEND
-; GFX1100_W32-NEXT: v_mov_b32_e32 v9, s4
+; GFX1100_W32-NEXT: v_mov_b32_e32 v8, s4
; GFX1100_W32-NEXT: s_mov_b32 s5, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W32-NEXT: s_mov_b32 s6, s5
@@ -883,10 +883,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX1100_W32-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
-; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX1100_W32-NEXT: buffer_load_format_x v9, v8, s[0:3], 0 idxen
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr9
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr8
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB2_1
; GFX1100_W32-NEXT: ; %bb.2:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s5
@@ -908,10 +908,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
-; GFX1100_W32-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
-; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX1100_W32-NEXT: buffer_load_format_x v9, v0, s[0:3], 0 idxen
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB2_4
; GFX1100_W32-NEXT: ; %bb.5:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
@@ -919,7 +919,7 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W32-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W32-NEXT: global_store_b32 v[11:12], v8, off dlc
+; GFX1100_W32-NEXT: global_store_b32 v[11:12], v9, off dlc
; GFX1100_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W32-NEXT: s_setpc_b64 s[30:31]
;
@@ -929,7 +929,7 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W64-NEXT: ;;#ASMSTART
; GFX1100_W64-NEXT: s_mov_b32 s4, 17
; GFX1100_W64-NEXT: ;;#ASMEND
-; GFX1100_W64-NEXT: v_mov_b32_e32 v9, s4
+; GFX1100_W64-NEXT: v_mov_b32_e32 v8, s4
; GFX1100_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W64-NEXT: s_mov_b64 s[8:9], s[6:7]
@@ -941,10 +941,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX1100_W64-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
-; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
+; GFX1100_W64-NEXT: buffer_load_format_x v9, v8, s[0:3], 0 idxen
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr9
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr8
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB2_1
; GFX1100_W64-NEXT: ; %bb.2:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[6:7]
@@ -966,10 +966,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
-; GFX1100_W64-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
-; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
+; GFX1100_W64-NEXT: buffer_load_format_x v9, v0, s[0:3], 0 idxen
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB2_4
; GFX1100_W64-NEXT: ; %bb.5:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
@@ -977,7 +977,7 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W64-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W64-NEXT: global_store_b32 v[11:12], v8, off dlc
+; GFX1100_W64-NEXT: global_store_b32 v[11:12], v9, off dlc
; GFX1100_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W64-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
index 6911408544888..6894f2ac9cf0e 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
@@ -48,9 +48,9 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W32-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
-; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB0_1
; GFX1010_W32-NEXT: ; %bb.2:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -73,9 +73,9 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W64-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
-; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB0_1
; GFX1010_W64-NEXT: ; %bb.2:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -99,9 +99,9 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W32-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
-; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB0_1
; GFX1100_W32-NEXT: ; %bb.2:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
@@ -124,9 +124,9 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W64-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
-; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB0_1
; GFX1100_W64-NEXT: ; %bb.2:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
@@ -308,8 +308,8 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W32-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
-; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB1_1
; GFX1010_W32-NEXT: ; %bb.2:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -325,9 +325,9 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; GFX1010_W32-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
-; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W32-NEXT: ; implicit-def: $vgpr8
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB1_3
; GFX1010_W32-NEXT: ; %bb.4:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -354,8 +354,8 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W64-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
-; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB1_1
; GFX1010_W64-NEXT: ; %bb.2:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -371,9 +371,9 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; GFX1010_W64-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
-; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W64-NEXT: ; implicit-def: $vgpr8
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB1_3
; GFX1010_W64-NEXT: ; %bb.4:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -401,8 +401,8 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W32-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
-; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB1_1
; GFX1100_W32-NEXT: ; %bb.2:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
@@ -418,9 +418,9 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
; GFX1100_W32-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
-; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W32-NEXT: ; implicit-def: $vgpr8
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB1_3
; GFX1100_W32-NEXT: ; %bb.4:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
@@ -447,8 +447,8 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W64-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
-; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB1_1
; GFX1100_W64-NEXT: ; %bb.2:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
@@ -464,9 +464,9 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
; GFX1100_W64-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
-; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W64-NEXT: ; implicit-def: $vgpr8
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB1_3
; GFX1100_W64-NEXT: ; %bb.4:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
@@ -779,7 +779,7 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W32-NEXT: ;;#ASMSTART
; GFX1010_W32-NEXT: s_mov_b32 s4, 17
; GFX1010_W32-NEXT: ;;#ASMEND
-; GFX1010_W32-NEXT: v_mov_b32_e32 v9, s4
+; GFX1010_W32-NEXT: v_mov_b32_e32 v8, s4
; GFX1010_W32-NEXT: s_mov_b32 s5, exec_lo
; GFX1010_W32-NEXT: s_mov_b32 s6, s5
; GFX1010_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
@@ -790,10 +790,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[8:9], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[10:11], v[2:3]
-; GFX1010_W32-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
-; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s6, s6
+; GFX1010_W32-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr9
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr8
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s6, s6
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB2_1
; GFX1010_W32-NEXT: ; %bb.2:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -814,10 +814,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
-; GFX1010_W32-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
-; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX1010_W32-NEXT: buffer_load_format_x v9, v0, s[4:7], 0 idxen
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB2_4
; GFX1010_W32-NEXT: ; %bb.5:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -825,7 +825,7 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W32-NEXT: .LBB2_6: ; %bb2
; GFX1010_W32-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W32-NEXT: global_store_dword v[11:12], v8, off
+; GFX1010_W32-NEXT: global_store_dword v[11:12], v9, off
; GFX1010_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W32-NEXT: s_setpc_b64 s[30:31]
;
@@ -835,7 +835,7 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W64-NEXT: ;;#ASMSTART
; GFX1010_W64-NEXT: s_mov_b32 s4, 17
; GFX1010_W64-NEXT: ;;#ASMEND
-; GFX1010_W64-NEXT: v_mov_b32_e32 v9, s4
+; GFX1010_W64-NEXT: v_mov_b32_e32 v8, s4
; GFX1010_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX1010_W64-NEXT: s_mov_b64 s[12:13], s[6:7]
; GFX1010_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
@@ -846,10 +846,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[8:9], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[10:11], v[2:3]
-; GFX1010_W64-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
-; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
+; GFX1010_W64-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr9
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr8
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB2_1
; GFX1010_W64-NEXT: ; %bb.2:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -870,10 +870,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
-; GFX1010_W64-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
-; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
+; GFX1010_W64-NEXT: buffer_load_format_x v9, v0, s[4:7], 0 idxen
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB2_4
; GFX1010_W64-NEXT: ; %bb.5:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -881,7 +881,7 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W64-NEXT: .LBB2_6: ; %bb2
; GFX1010_W64-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W64-NEXT: global_store_dword v[11:12], v8, off
+; GFX1010_W64-NEXT: global_store_dword v[11:12], v9, off
; GFX1010_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W64-NEXT: s_setpc_b64 s[30:31]
;
@@ -891,7 +891,7 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W32-NEXT: ;;#ASMSTART
; GFX1100_W32-NEXT: s_mov_b32 s4, 17
; GFX1100_W32-NEXT: ;;#ASMEND
-; GFX1100_W32-NEXT: v_mov_b32_e32 v9, s4
+; GFX1100_W32-NEXT: v_mov_b32_e32 v8, s4
; GFX1100_W32-NEXT: s_mov_b32 s5, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W32-NEXT: s_mov_b32 s6, s5
@@ -903,10 +903,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX1100_W32-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
-; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX1100_W32-NEXT: buffer_load_format_x v9, v8, s[0:3], 0 idxen
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr9
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr8
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB2_1
; GFX1100_W32-NEXT: ; %bb.2:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s5
@@ -928,10 +928,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
-; GFX1100_W32-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
-; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX1100_W32-NEXT: buffer_load_format_x v9, v0, s[0:3], 0 idxen
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB2_4
; GFX1100_W32-NEXT: ; %bb.5:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
@@ -939,7 +939,7 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W32-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W32-NEXT: global_store_b32 v[11:12], v8, off dlc
+; GFX1100_W32-NEXT: global_store_b32 v[11:12], v9, off dlc
; GFX1100_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W32-NEXT: s_setpc_b64 s[30:31]
;
@@ -949,7 +949,7 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W64-NEXT: ;;#ASMSTART
; GFX1100_W64-NEXT: s_mov_b32 s4, 17
; GFX1100_W64-NEXT: ;;#ASMEND
-; GFX1100_W64-NEXT: v_mov_b32_e32 v9, s4
+; GFX1100_W64-NEXT: v_mov_b32_e32 v8, s4
; GFX1100_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W64-NEXT: s_mov_b64 s[8:9], s[6:7]
@@ -961,10 +961,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX1100_W64-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
-; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
+; GFX1100_W64-NEXT: buffer_load_format_x v9, v8, s[0:3], 0 idxen
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr9
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr8
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB2_1
; GFX1100_W64-NEXT: ; %bb.2:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[6:7]
@@ -986,10 +986,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
-; GFX1100_W64-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
-; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
+; GFX1100_W64-NEXT: buffer_load_format_x v9, v0, s[0:3], 0 idxen
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB2_4
; GFX1100_W64-NEXT: ; %bb.5:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
@@ -997,7 +997,7 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W64-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W64-NEXT: global_store_b32 v[11:12], v8, off dlc
+; GFX1100_W64-NEXT: global_store_b32 v[11:12], v9, off dlc
; GFX1100_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W64-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir
index 37d73ddd1b3d7..29ab3c8895af3 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir
@@ -101,7 +101,7 @@ body: |
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W64-NEXT: {{ $}}
; W64-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W64-NEXT: [[S_ANDN2_WREXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_ANDN2_WREXEC_B64 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W64-NEXT: [[S_ANDN2_WREXEC_B64_term:%[0-9]+]]:sreg_64_xexec = S_ANDN2_WREXEC_B64_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; W64-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .3:
@@ -142,7 +142,7 @@ body: |
; W32-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W32-NEXT: {{ $}}
; W32-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W32-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W32-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; W32-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .3:
@@ -251,7 +251,7 @@ body: |
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W64-NEXT: {{ $}}
; W64-NEXT: [[BUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W64-NEXT: [[S_ANDN2_WREXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_ANDN2_WREXEC_B64 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W64-NEXT: [[S_ANDN2_WREXEC_B64_term:%[0-9]+]]:sreg_64_xexec = S_ANDN2_WREXEC_B64_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; W64-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .3:
@@ -292,7 +292,7 @@ body: |
; W32-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W32-NEXT: {{ $}}
; W32-NEXT: [[BUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W32-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W32-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; W32-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .3:
@@ -402,7 +402,7 @@ body: |
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W64-NEXT: {{ $}}
; W64-NEXT: [[BUFFER_LOAD_FORMAT_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_BOTHEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W64-NEXT: [[S_ANDN2_WREXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_ANDN2_WREXEC_B64 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W64-NEXT: [[S_ANDN2_WREXEC_B64_term:%[0-9]+]]:sreg_64_xexec = S_ANDN2_WREXEC_B64_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; W64-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .3:
@@ -443,7 +443,7 @@ body: |
; W32-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W32-NEXT: {{ $}}
; W32-NEXT: [[BUFFER_LOAD_FORMAT_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_BOTHEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W32-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W32-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; W32-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .3:
@@ -671,7 +671,7 @@ body: |
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W64-NEXT: {{ $}}
; W64-NEXT: [[BUFFER_LOAD_FORMAT_X_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFSET killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W64-NEXT: [[S_ANDN2_WREXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_ANDN2_WREXEC_B64 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W64-NEXT: [[S_ANDN2_WREXEC_B64_term:%[0-9]+]]:sreg_64_xexec = S_ANDN2_WREXEC_B64_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; W64-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .3:
@@ -712,7 +712,7 @@ body: |
; W32-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W32-NEXT: {{ $}}
; W32-NEXT: [[BUFFER_LOAD_FORMAT_X_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFSET killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W32-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W32-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; W32-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .3:
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll b/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
index 960b6e61f7c83..ba0024ff16dea 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
@@ -27,8 +27,8 @@ define void @vgpr_descriptor_waterfall_loop_idom_update(ptr %arg) #0 {
; GCN-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[2:3]
; GCN-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[4:5]
; GCN-NEXT: buffer_store_dword v0, v0, s[4:7], 0 offen
-; GCN-NEXT: s_andn2_wrexec_b32 s9, s9
; GCN-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GCN-NEXT: s_andn2_wrexec_b32 s9, s9
; GCN-NEXT: s_cbranch_execnz .LBB0_2
; GCN-NEXT: ; %bb.3: ; in Loop: Header=BB0_1 Depth=1
; GCN-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -60,8 +60,8 @@ define void @vgpr_descriptor_waterfall_loop_idom_update(ptr %arg) #0 {
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
; GFX11-NEXT: buffer_store_b32 v0, v0, s[0:3], 0 offen
-; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: s_cbranch_execnz .LBB0_2
; GFX11-NEXT: ; %bb.3: ; in Loop: Header=BB0_1 Depth=1
; GFX11-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
index 77382ba781d5a..577c8a28e42db 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
@@ -266,7 +266,7 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed [[REG_SEQUENCE1]], 0, csr_amdgpu_si_gfx, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed $vgpr0, implicit-def $vgpr0
; SI-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
- ; SI-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 killed [[PHI4]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; SI-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term killed [[PHI4]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: SI_WATERFALL_LOOP %bb.3, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.5:
@@ -303,7 +303,7 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed [[REG_SEQUENCE3]], 0, csr_amdgpu_si_gfx, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed $vgpr0, implicit-def $vgpr0
; SI-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
- ; SI-NEXT: [[S_ANDN2_WREXEC_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 killed [[PHI7]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; SI-NEXT: [[S_ANDN2_WREXEC_B32_term1:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term killed [[PHI7]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: SI_WATERFALL_LOOP %bb.7, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.9:
@@ -387,7 +387,7 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed [[REG_SEQUENCE1]], 0, csr_amdgpu_si_gfx, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed $vgpr0, implicit-def $vgpr0
; SI-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
- ; SI-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 killed [[PHI3]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; SI-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term killed [[PHI3]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: SI_WATERFALL_LOOP %bb.3, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.5:
@@ -423,7 +423,7 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed [[REG_SEQUENCE3]], 0, csr_amdgpu_si_gfx, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed $vgpr0, implicit-def $vgpr0
; SI-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
- ; SI-NEXT: [[S_ANDN2_WREXEC_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 killed [[PHI5]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; SI-NEXT: [[S_ANDN2_WREXEC_B32_term1:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term killed [[PHI5]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: SI_WATERFALL_LOOP %bb.7, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.9:
@@ -624,14 +624,14 @@ define protected amdgpu_kernel void @nested_waterfalls(ptr addrspace(1) %tex.coe
; SI-NEXT: successors: %bb.4(0x40000000), %bb.6(0x40000000)
; SI-NEXT: {{ $}}
; SI-NEXT: [[IMAGE_SAMPLE_V1_V2_nsa_gfx10_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V2_nsa_gfx10 undef %29:vgpr_32, undef %31:vgpr_32, [[REG_SEQUENCE5]], killed [[REG_SEQUENCE8]], 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; SI-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 killed [[PHI1]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; SI-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term killed [[PHI1]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: SI_WATERFALL_LOOP %bb.4, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.6:
; SI-NEXT: successors: %bb.2(0x40000000), %bb.7(0x40000000)
; SI-NEXT: {{ $}}
; SI-NEXT: $exec_lo = S_MOV_B32 killed [[S_MOV_B32_1]]
- ; SI-NEXT: [[S_ANDN2_WREXEC_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 killed [[PHI]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; SI-NEXT: [[S_ANDN2_WREXEC_B32_term1:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term killed [[PHI]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.7:
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll b/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll
index cdec79cc385e4..ab9ae948d48de 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll
@@ -187,9 +187,9 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: s_mov_b64 s[2:3], s[14:15]
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
; SI-NEXT: v_mov_b32_e32 v1, v0
-; SI-NEXT: s_andn2_wrexec_b32 s8, s8
; SI-NEXT: ; implicit-def: $vgpr4_vgpr5
; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: s_andn2_wrexec_b32 s8, s8
; SI-NEXT: s_cbranch_execnz .LBB3_2
; SI-NEXT: ; %bb.3:
; SI-NEXT: s_mov_b32 exec_lo, s7
@@ -210,9 +210,9 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: s_mov_b64 s[2:3], s[14:15]
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
; SI-NEXT: v_mov_b32_e32 v1, v0
-; SI-NEXT: s_andn2_wrexec_b32 s8, s8
; SI-NEXT: ; implicit-def: $vgpr2_vgpr3
; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: s_andn2_wrexec_b32 s8, s8
; SI-NEXT: s_cbranch_execnz .LBB3_6
; SI-NEXT: ; %bb.7:
; SI-NEXT: s_mov_b32 exec_lo, s7
@@ -266,8 +266,8 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: s_mov_b64 s[0:1], s[12:13]
; SI-NEXT: s_mov_b64 s[2:3], s[14:15]
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
-; SI-NEXT: s_andn2_wrexec_b32 s8, s8
; SI-NEXT: ; implicit-def: $vgpr4_vgpr5
+; SI-NEXT: s_andn2_wrexec_b32 s8, s8
; SI-NEXT: s_cbranch_execnz .LBB4_2
; SI-NEXT: ; %bb.3:
; SI-NEXT: s_mov_b32 exec_lo, s7
@@ -287,8 +287,8 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: s_mov_b64 s[0:1], s[12:13]
; SI-NEXT: s_mov_b64 s[2:3], s[14:15]
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
-; SI-NEXT: s_andn2_wrexec_b32 s8, s8
; SI-NEXT: ; implicit-def: $vgpr2_vgpr3
+; SI-NEXT: s_andn2_wrexec_b32 s8, s8
; SI-NEXT: s_cbranch_execnz .LBB4_6
; SI-NEXT: ; %bb.7:
; SI-NEXT: s_mov_b32 exec_lo, s7
diff --git a/llvm/test/CodeGen/AMDGPU/waterfall-loop-wrexec-terminator.mir b/llvm/test/CodeGen/AMDGPU/waterfall-loop-wrexec-terminator.mir
new file mode 100644
index 0000000000000..817684257b274
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/waterfall-loop-wrexec-terminator.mir
@@ -0,0 +1,150 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu10.10 -mattr=+wavefrontsize32 -verify-machineinstrs --run-pass=si-fix-sgpr-copies,livevars,phi-node-elimination -o - %s | FileCheck %s --check-prefixes=W32
+# RUN: llc -mtriple=amdgpu10.10 -mattr=+wavefrontsize64 -verify-machineinstrs --run-pass=si-fix-sgpr-copies,livevars,phi-node-elimination -o - %s | FileCheck %s --check-prefixes=W64
+# RUN: llc -mtriple=amdgpu10.10 -mattr=+wavefrontsize32 -O0 -verify-machineinstrs --run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s --check-prefixes=O0
+
+# The waterfall loop EXEC update is a terminator, so PHI elimination has to
+# fold the exec PHI into its def instead of inserting a COPY after it. At -O0
+# the plain, non-terminator form is used because RegAllocFast would place the
+# spill of the def after the terminator.
+
+---
+name: waterfall_wrexec_term
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0', virtual-reg: '%0' }
+ - { reg: '$vgpr1', virtual-reg: '%1' }
+ - { reg: '$vgpr2', virtual-reg: '%2' }
+ - { reg: '$vgpr3', virtual-reg: '%3' }
+ - { reg: '$vgpr4', virtual-reg: '%4' }
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; W32-LABEL: name: waterfall_wrexec_term
+ ; W32: successors: %bb.1(0x80000000)
+ ; W32-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; W32-NEXT: {{ $}}
+ ; W32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY killed $vgpr4
+ ; W32-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY killed $vgpr3
+ ; W32-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY killed $vgpr2
+ ; W32-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY killed $vgpr1
+ ; W32-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
+ ; W32-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE killed [[COPY4]], %subreg.sub0, killed [[COPY3]], %subreg.sub1, killed [[COPY2]], %subreg.sub2, killed [[COPY1]], %subreg.sub3
+ ; W32-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; W32-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; W32-NEXT: [[COPY5:%[0-9]+]]:sreg_32_xm0_xexec = COPY killed [[S_MOV_B32_1]]
+ ; W32-NEXT: {{ $}}
+ ; W32-NEXT: .1:
+ ; W32-NEXT: successors: %bb.2(0x80000000)
+ ; W32-NEXT: {{ $}}
+ ; W32-NEXT: [[COPY6:%[0-9]+]]:sreg_32_xm0_xexec = COPY killed [[COPY5]]
+ ; W32-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+ ; W32-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+ ; W32-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; W32-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+ ; W32-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+ ; W32-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; W32-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_1]], %subreg.sub1, killed [[V_READFIRSTLANE_B32_2]], %subreg.sub2, killed [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32_term killed [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32_term killed [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; W32-NEXT: {{ $}}
+ ; W32-NEXT: .2:
+ ; W32-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; W32-NEXT: {{ $}}
+ ; W32-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[COPY]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
+ ; W32-NEXT: [[COPY5:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term killed [[COPY6]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; W32-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; W32-NEXT: {{ $}}
+ ; W32-NEXT: .3:
+ ; W32-NEXT: $exec_lo = S_MOV_B32 killed [[S_MOV_B32_]]
+ ; W32-NEXT: $vgpr0 = COPY killed [[BUFFER_LOAD_FORMAT_X_IDXEN]]
+ ; W32-NEXT: S_ENDPGM 0, implicit killed $vgpr0
+ ;
+ ; W64-LABEL: name: waterfall_wrexec_term
+ ; W64: successors: %bb.1(0x80000000)
+ ; W64-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; W64-NEXT: {{ $}}
+ ; W64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY killed $vgpr4
+ ; W64-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY killed $vgpr3
+ ; W64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY killed $vgpr2
+ ; W64-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY killed $vgpr1
+ ; W64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
+ ; W64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE killed [[COPY4]], %subreg.sub0, killed [[COPY3]], %subreg.sub1, killed [[COPY2]], %subreg.sub2, killed [[COPY1]], %subreg.sub3
+ ; W64-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; W64-NEXT: [[S_MOV_B64_1:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; W64-NEXT: [[COPY5:%[0-9]+]]:sreg_64_xexec = COPY killed [[S_MOV_B64_1]]
+ ; W64-NEXT: {{ $}}
+ ; W64-NEXT: .1:
+ ; W64-NEXT: successors: %bb.2(0x80000000)
+ ; W64-NEXT: {{ $}}
+ ; W64-NEXT: [[COPY6:%[0-9]+]]:sreg_64_xexec = COPY killed [[COPY5]]
+ ; W64-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+ ; W64-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+ ; W64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; W64-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+ ; W64-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+ ; W64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; W64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_1]], %subreg.sub1, killed [[V_READFIRSTLANE_B32_2]], %subreg.sub2, killed [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e32_term killed [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e32_term killed [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; W64-NEXT: {{ $}}
+ ; W64-NEXT: .2:
+ ; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; W64-NEXT: {{ $}}
+ ; W64-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[COPY]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
+ ; W64-NEXT: [[COPY5:%[0-9]+]]:sreg_64_xexec = S_ANDN2_WREXEC_B64_term killed [[COPY6]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; W64-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; W64-NEXT: {{ $}}
+ ; W64-NEXT: .3:
+ ; W64-NEXT: $exec = S_MOV_B64 killed [[S_MOV_B64_]]
+ ; W64-NEXT: $vgpr0 = COPY killed [[BUFFER_LOAD_FORMAT_X_IDXEN]]
+ ; W64-NEXT: S_ENDPGM 0, implicit killed $vgpr0
+ ;
+ ; O0-LABEL: name: waterfall_wrexec_term
+ ; O0: successors: %bb.1(0x80000000)
+ ; O0-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; O0-NEXT: {{ $}}
+ ; O0-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; O0-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; O0-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; O0-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; O0-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; O0-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
+ ; O0-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; O0-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; O0-NEXT: {{ $}}
+ ; O0-NEXT: .1:
+ ; O0-NEXT: successors: %bb.2(0x80000000)
+ ; O0-NEXT: {{ $}}
+ ; O0-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %14, %bb.2
+ ; O0-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+ ; O0-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+ ; O0-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; O0-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+ ; O0-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+ ; O0-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; O0-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; O0-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; O0-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; O0-NEXT: {{ $}}
+ ; O0-NEXT: .2:
+ ; O0-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; O0-NEXT: {{ $}}
+ ; O0-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[COPY]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
+ ; O0-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; O0-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; O0-NEXT: {{ $}}
+ ; O0-NEXT: .3:
+ ; O0-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; O0-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_IDXEN]]
+ ; O0-NEXT: S_ENDPGM 0, implicit $vgpr0
+ %4:vgpr_32 = COPY $vgpr4
+ %3:vgpr_32 = COPY $vgpr3
+ %2:vgpr_32 = COPY $vgpr2
+ %1:vgpr_32 = COPY $vgpr1
+ %0:vgpr_32 = COPY $vgpr0
+ %5:sgpr_128 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1, %2, %subreg.sub2, %3, %subreg.sub3
+ %6:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN %4, killed %5, 0, 0, 0, 0, implicit $exec
+ $vgpr0 = COPY %6
+ S_ENDPGM 0, implicit $vgpr0
+...
>From 3753bef06632c4b6d9689b711e218c4c0449c86d Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Tue, 8 Sep 2026 16:05:04 +0200
Subject: [PATCH 2/3] rework
---
llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h | 3 -
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 27 ++--
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 6 -
llvm/lib/Target/AMDGPU/SIInstructions.td | 2 -
.../Target/AMDGPU/SIOptimizeExecMasking.cpp | 98 +++++++++++-
.../buffer-fat-pointer-atomicrmw-fadd.ll | 8 +-
.../buffer-fat-pointer-atomicrmw-fmax.ll | 8 +-
.../buffer-fat-pointer-atomicrmw-fmin.ll | 8 +-
...e92561-restore-undef-scc-verifier-error.ll | 2 +-
.../legalize-amdgcn.raw.ptr.buffer.load.ll | 108 +++++++++-----
.../AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll | 2 +-
.../AMDGPU/llvm.amdgcn.dual_intersect_ray.ll | 2 +-
...mdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll | 6 +-
....amdgcn.struct.buffer.load.format.v3f16.ll | 6 +-
...cn.struct.ptr.buffer.atomic.fadd.v2bf16.ll | 4 +-
...gcn.struct.ptr.buffer.atomic.fadd_nortn.ll | 12 +-
...mdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll | 12 +-
...mdgcn.struct.ptr.buffer.atomic.fmax.f32.ll | 20 +--
...mdgcn.struct.ptr.buffer.atomic.fmin.f32.ll | 20 +--
...gcn.struct.ptr.buffer.load.format.v3f16.ll | 4 +-
llvm/test/CodeGen/AMDGPU/mixed-vmem-types.ll | 4 +-
.../AMDGPU/move-to-valu-vimage-vsample.ll | 18 ++-
...uf-legalize-operands-non-ptr-intrinsics.ll | 80 +++++-----
.../CodeGen/AMDGPU/mubuf-legalize-operands.ll | 80 +++++-----
.../AMDGPU/mubuf-legalize-operands.mir | 24 ++-
.../si-optimize-exec-masking-andn2-wrexec.mir | 140 ++++++++++++++++++
...r-descriptor-waterfall-loop-idom-update.ll | 4 +-
llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll | 18 ++-
llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll | 8 +-
...waterfall-loop-exec-update-terminator.mir} | 65 ++------
30 files changed, 513 insertions(+), 286 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/si-optimize-exec-masking-andn2-wrexec.mir
rename llvm/test/CodeGen/AMDGPU/{waterfall-loop-wrexec-terminator.mir => waterfall-loop-exec-update-terminator.mir} (61%)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h b/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
index fc1c005f25426..af14066c1bd46 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
@@ -28,7 +28,6 @@ class LaneMaskConstants {
const unsigned AndN2SaveExecOpc;
const unsigned AndN2TermOpc;
const unsigned AndN2WrExecOpc; // GFX10+ (HasNoSdstCMPX) only
- const unsigned AndN2WrExecTermOpc;
const unsigned AndSaveExecOpc;
const unsigned AndSaveExecTermOpc;
const unsigned BfmOpc;
@@ -56,8 +55,6 @@ class LaneMaskConstants {
: AMDGPU::S_ANDN2_B64_term),
AndN2WrExecOpc(IsWave32 ? AMDGPU::S_ANDN2_WREXEC_B32
: AMDGPU::S_ANDN2_WREXEC_B64),
- AndN2WrExecTermOpc(IsWave32 ? AMDGPU::S_ANDN2_WREXEC_B32_term
- : AMDGPU::S_ANDN2_WREXEC_B64_term),
AndSaveExecOpc(IsWave32 ? AMDGPU::S_AND_SAVEEXEC_B32
: AMDGPU::S_AND_SAVEEXEC_B64),
AndSaveExecTermOpc(IsWave32 ? AMDGPU::S_AND_SAVEEXEC_B32_term
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index ff5ce9314576d..62edbcccdf0b4 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -2104,13 +2104,6 @@ bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
MI.setDesc(get(AMDGPU::V_CMPX_EQ_U64_nosdst_e32));
break;
- case AMDGPU::S_ANDN2_WREXEC_B64_term:
- MI.setDesc(get(AMDGPU::S_ANDN2_WREXEC_B64));
- break;
- case AMDGPU::S_ANDN2_WREXEC_B32_term:
- MI.setDesc(get(AMDGPU::S_ANDN2_WREXEC_B32));
- break;
-
case AMDGPU::SI_SPILL_S32_TO_VGPR:
MI.setDesc(get(AMDGPU::V_WRITELANE_B32));
break;
@@ -3262,8 +3255,6 @@ bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
case AMDGPU::S_AND_SAVEEXEC_B32_term:
case AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term:
case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
- case AMDGPU::S_ANDN2_WREXEC_B32_term:
- case AMDGPU::S_ANDN2_WREXEC_B64_term:
break;
case AMDGPU::SI_IF:
case AMDGPU::SI_ELSE:
@@ -7500,15 +7491,17 @@ static void emitLoadScalarOpsFromVGPRLoop(
I = BodyBB.end();
if (UseNewExecInstructions) {
- // Terminator form lets PHI elimination fold this into the exec PHI's
- // def. Skip it at -O0: RegAllocFast spills live-out defs right after
- // them, not at the first terminator.
- bool UseTermForm = MF.getTarget().getOptLevel() != CodeGenOptLevel::None;
+ // Compute the remaining lanes into a plain virtual register and write EXEC
+ // from a terminator, so spill code for NewExec is placed before EXEC
+ // changes. SIOptimizeExecMasking opportunistically folds the pair back
+ // into S_ANDN2_WREXEC after register allocation; if it can't, this is
+ // still correct, just one instruction longer.
MRI.setSimpleHint(NewExec, PhiExec);
- BuildMI(BodyBB, I, DL,
- TII.get(UseTermForm ? LMC.AndN2WrExecTermOpc : LMC.AndN2WrExecOpc),
- NewExec)
- .addReg(PhiExec);
+ BuildMI(BodyBB, I, DL, TII.get(LMC.AndN2Opc), NewExec)
+ .addReg(PhiExec)
+ .addReg(LMC.ExecReg);
+ BuildMI(BodyBB, I, DL, TII.get(LMC.MovTermOpc), LMC.ExecReg)
+ .addReg(NewExec);
} else {
// Update EXEC, switch all done bits to 0 and all todo bits to 1.
BuildMI(BodyBB, I, DL, TII.get(LMC.XorTermOpc), LMC.ExecReg)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 98381005b3c86..4bf0f557ae4e1 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -351,12 +351,6 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
bool expandPostRAPseudo(MachineInstr &MI) const override;
- // See the comment in emitLoadScalarOpsFromVGPRLoop.
- bool isUnspillableTerminatorImpl(const MachineInstr *MI) const override {
- return MI->getOpcode() == AMDGPU::S_ANDN2_WREXEC_B32_term ||
- MI->getOpcode() == AMDGPU::S_ANDN2_WREXEC_B64_term;
- }
-
void
reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
Register DestReg, unsigned SubIdx, const MachineInstr &Orig,
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index a45d424a6a882..eb23b6fea26b3 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -497,7 +497,6 @@ def S_OR_B64_term : WrapTerminatorInst<S_OR_B64>;
def S_ANDN2_B64_term : WrapTerminatorInst<S_ANDN2_B64>;
def S_AND_B64_term : WrapTerminatorInst<S_AND_B64>;
def S_AND_SAVEEXEC_B64_term : WrapTerminatorInst<S_AND_SAVEEXEC_B64>;
-def S_ANDN2_WREXEC_B64_term : WrapTerminatorInst<S_ANDN2_WREXEC_B64>;
}
let WaveSizePredicate = isWave32 in {
@@ -507,7 +506,6 @@ def S_OR_B32_term : WrapTerminatorInst<S_OR_B32>;
def S_ANDN2_B32_term : WrapTerminatorInst<S_ANDN2_B32>;
def S_AND_B32_term : WrapTerminatorInst<S_AND_B32>;
def S_AND_SAVEEXEC_B32_term : WrapTerminatorInst<S_AND_SAVEEXEC_B32>;
-def S_ANDN2_WREXEC_B32_term : WrapTerminatorInst<S_ANDN2_WREXEC_B32>;
}
class WrapTerminatorVOPC<VOPC_Pseudo base_inst> : VPseudoInstSI<
diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
index fab31365eca17..245122e141a9f 100644
--- a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
+++ b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
@@ -64,6 +64,8 @@ class SIOptimizeExecMasking {
SmallVectorImpl<MachineOperand *> *KillFlagCandidates = nullptr,
unsigned MaxInstructions = 20) const;
bool optimizeExecSequence();
+ bool optimizeAndN2WrExecSequence(MachineInstr &CopyToExecInst,
+ Register Dst) const;
void tryRecordVCmpxAndSaveexecSequence(MachineInstr &MI);
bool optimizeVCMPSaveExecSequence(MachineInstr &SaveExecInstr,
MachineInstr &VCmp) const;
@@ -299,12 +301,6 @@ bool SIOptimizeExecMasking::removeTerminatorBit(MachineInstr &MI) const {
case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
MI.setDesc(TII->get(AMDGPU::V_CMPX_EQ_U64_nosdst_e32));
return true;
- case AMDGPU::S_ANDN2_WREXEC_B64_term:
- MI.setDesc(TII->get(AMDGPU::S_ANDN2_WREXEC_B64));
- return true;
- case AMDGPU::S_ANDN2_WREXEC_B32_term:
- MI.setDesc(TII->get(AMDGPU::S_ANDN2_WREXEC_B32));
- return true;
default:
return false;
}
@@ -494,6 +490,8 @@ bool SIOptimizeExecMasking::optimizeExecSequence() {
CopyToExecInst->eraseFromParent();
Changed = true;
+ } else if (optimizeAndN2WrExecSequence(*CopyToExecInst, CopyToExec)) {
+ Changed = true;
}
continue;
@@ -610,6 +608,94 @@ bool SIOptimizeExecMasking::optimizeExecSequence() {
return Changed;
}
+// Fold
+//
+// sdst = S_ANDN2_B32 ssrc, exec
+// exec = COPY sdst
+// =>
+// sdst = S_ANDN2_WREXEC_B32 ssrc
+//
+// The waterfall loop emits the two operations separately so that spill code
+// for sdst can be inserted before exec is narrowed.
+bool SIOptimizeExecMasking::optimizeAndN2WrExecSequence(
+ MachineInstr &CopyToExecInst, Register Dst) const {
+ if (!ST->hasNoSdstCMPX() || TII->pseudoToMCOpcode(LMC.AndN2WrExecOpc) == -1)
+ return false;
+
+ MachineBasicBlock &MBB = *CopyToExecInst.getParent();
+
+ // Keep the fused instruction ahead of any trailing meta instructions (e.g.
+ // DBG_VALUEs of Dst), which emit no code, so that the sequence is unchanged
+ // in the common case where the two are already adjacent.
+ MachineBasicBlock::iterator InsertPt = CopyToExecInst.getIterator();
+ while (InsertPt != MBB.begin() && std::prev(InsertPt)->isMetaInstruction())
+ --InsertPt;
+
+ // Scan back for the s_andn2 computing the new exec value. The scheduler may
+ // have moved it away from the exec write, so allow instructions in between
+ // as long as sinking the s_andn2 past them is safe.
+ const unsigned SearchLimit = 20;
+ unsigned Count = 0;
+ MachineInstr *AndN2Inst = nullptr;
+ for (MachineInstr &MI : make_range(
+ std::next(MachineBasicBlock::reverse_iterator(CopyToExecInst)),
+ MBB.rend())) {
+ if (MI.isMetaInstruction())
+ continue;
+ if (++Count > SearchLimit)
+ return false;
+ if (MI.getOpcode() == LMC.AndN2Opc && MI.getOperand(0).getReg() == Dst) {
+ AndN2Inst = &MI;
+ break;
+ }
+ bool ReadsDst = false, ModifiesDst = false, ModifiesExec = false,
+ ReadsSCC = false;
+ for (const MachineOperand &MO : MI.operands()) {
+ if (!MO.isReg())
+ continue;
+ if (TRI->regsOverlap(MO.getReg(), Dst)) {
+ if (MO.isUse())
+ ReadsDst = true;
+ else
+ ModifiesDst = true;
+ }
+ if (!MO.isUse() && TRI->regsOverlap(MO.getReg(), LMC.ExecReg))
+ ModifiesExec = true;
+ if (MO.isUse() && TRI->regsOverlap(MO.getReg(), AMDGPU::SCC))
+ ReadsSCC = true;
+ }
+ if (ReadsDst || ModifiesDst || ModifiesExec || ReadsSCC)
+ return false;
+ }
+
+ if (!AndN2Inst)
+ return false;
+
+ const MachineOperand &Src = AndN2Inst->getOperand(1);
+ const MachineOperand &Mask = AndN2Inst->getOperand(2);
+ if (!Src.isReg() || !Mask.isReg() || Mask.getReg() != LMC.ExecReg)
+ return false;
+
+ for (MachineInstr &MI :
+ make_range(std::next(MachineBasicBlock::iterator(AndN2Inst)), InsertPt))
+ if (MI.modifiesRegister(Src.getReg(), TRI))
+ return false;
+
+ // The fused form also clobbers SCC, at the point the s_andn2 is moved to.
+ if (isRegisterInUseAfter(CopyToExecInst, AMDGPU::SCC))
+ return false;
+
+ LLVM_DEBUG(dbgs() << "Fold exec write: " << *AndN2Inst);
+
+ BuildMI(MBB, InsertPt, AndN2Inst->getDebugLoc(), TII->get(LMC.AndN2WrExecOpc),
+ Dst)
+ .addReg(Src.getReg());
+
+ AndN2Inst->eraseFromParent();
+ CopyToExecInst.eraseFromParent();
+ return true;
+}
+
// Inserts the optimized s_mov_b32 / v_cmpx sequence based on the
// operands extracted from a v_cmp ..., s_and_saveexec pattern.
bool SIOptimizeExecMasking::optimizeVCMPSaveExecSequence(
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index b2cf394ccf193..4719d6edbaaa5 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -387,9 +387,9 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_add_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
-; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: s_cbranch_execnz .LBB2_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
@@ -444,9 +444,9 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_add_f32 v5, v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-NEXT: ; implicit-def: $vgpr4
-; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: s_cbranch_execnz .LBB2_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s4
@@ -7036,9 +7036,9 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_pk_add_f16 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
-; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
@@ -9857,9 +9857,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_pk_add_bf16 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
-; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: s_cbranch_execnz .LBB28_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index 3fdd9467a74f2..cecc14be3c521 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -378,9 +378,9 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_max_num_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
-; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: s_cbranch_execnz .LBB2_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
@@ -466,9 +466,9 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_max_f32 v5, v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-NEXT: ; implicit-def: $vgpr4
-; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: s_cbranch_execnz .LBB2_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s4
@@ -494,9 +494,9 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmax v5, v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
-; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB2_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -1710,9 +1710,9 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmax_x2 v[5:6], v4, s[4:7], 0 offen offset:2048 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
-; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB7_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index c6cd077701cdd..7859ac4841a10 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -378,9 +378,9 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_min_num_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
-; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: s_cbranch_execnz .LBB2_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
@@ -466,9 +466,9 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_min_f32 v5, v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-NEXT: ; implicit-def: $vgpr4
-; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: s_cbranch_execnz .LBB2_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s4
@@ -494,9 +494,9 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmin v5, v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
-; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB2_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -1710,9 +1710,9 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmin_x2 v[5:6], v4, s[4:7], 0 offen offset:2048 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
-; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB7_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
diff --git a/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll b/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
index e22aeee59af7a..124ba142a56f6 100644
--- a/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
+++ b/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
@@ -37,8 +37,8 @@ define void @issue92561(ptr addrspace(1) %arg) {
; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4)
; SDAG-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; SDAG-NEXT: image_sample_c_lz v9, [v8, v8, v8, v8], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
-; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; SDAG-NEXT: s_and_not1_wrexec_b32 s13, s13
+; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; SDAG-NEXT: s_cbranch_execnz .LBB0_1
; SDAG-NEXT: ; %bb.2:
; SDAG-NEXT: s_mov_b32 exec_lo, s12
diff --git a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll
index 3c7394d9753ca..1ad2f1079b1bb 100644
--- a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll
@@ -95,7 +95,8 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(ptr
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -199,7 +200,8 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__sgpr_voffset__sgpr_soffset(ptr
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -303,7 +305,8 @@ define float @raw_ptr_buffer_load_f32__vgpr_rsrc__vgpr_voffset__sgpr_soffset(ptr
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -407,7 +410,8 @@ define float @raw_ptr_buffer_load_f32__vgpr_rsrc__vgpr_voffset__vgpr_soffset(ptr
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -511,7 +515,8 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 1, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -615,7 +620,8 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 2, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -719,7 +725,8 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 4, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -823,7 +830,8 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 6, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -927,7 +935,8 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 5, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -1031,7 +1040,8 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 7, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -1138,7 +1148,8 @@ define <2 x float> @raw_ptr_buffer_load_v2f32__sgpr_rsrc__vgpr_voffset__sgpr_sof
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s64) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -1249,7 +1260,8 @@ define <3 x float> @raw_ptr_buffer_load_v3f32__sgpr_rsrc__vgpr_voffset__sgpr_sof
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN:%[0-9]+]]:vreg_96_align2 = BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s96) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -1364,7 +1376,8 @@ define <4 x float> @raw_ptr_buffer_load_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_sof
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -1475,7 +1488,8 @@ define half @raw_ptr_buffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(ptr
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -1579,7 +1593,8 @@ define <2 x half> @raw_ptr_buffer_load_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -1685,7 +1700,8 @@ define <4 x half> @raw_ptr_buffer_load_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s64) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -1791,7 +1807,8 @@ define float @raw_ptr_buffer_load_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffset_zext
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_UBYTE_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8) from %ir.rsrc, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -1896,7 +1913,8 @@ define float @raw_ptr_buffer_load_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sext
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_SBYTE_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SBYTE_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8) from %ir.rsrc, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -2001,7 +2019,8 @@ define float @raw_ptr_buffer_load_i16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_zex
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -2106,7 +2125,8 @@ define float @raw_ptr_buffer_load_i16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sex
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_SSHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SSHORT_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -2213,7 +2233,8 @@ define half @raw_ptr_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset(ptr
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -2318,7 +2339,8 @@ define float @raw_ptr_buffer_load_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffset(ptr
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_UBYTE_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8) from %ir.rsrc, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -2421,7 +2443,8 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vdpr_voffset__sgpr_soffset__vo
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -2522,7 +2545,8 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__vo
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -2624,7 +2648,8 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__vo
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 4096, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -2721,7 +2746,8 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset0(ptr a
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE5]], $sgpr_null, 8388604, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -2826,7 +2852,8 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_vof
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -2932,7 +2959,8 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__vo
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -3040,7 +3068,8 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__vo
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -3139,7 +3168,8 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sof
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY]], killed [[REG_SEQUENCE5]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -3237,7 +3267,8 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sof
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY]], killed [[REG_SEQUENCE5]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -3344,7 +3375,8 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sof
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -3452,7 +3484,8 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sof
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -3560,7 +3593,8 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sof
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -3669,7 +3703,8 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sof
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
@@ -3778,7 +3813,8 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_vof
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll
index f686e13303d49..7aa4816d26413 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll
@@ -180,11 +180,11 @@ define amdgpu_ps <10 x float> @image_bvh8_intersect_ray_vvvvvv(i64 %node_ptr, fl
; GFX12-SDAG-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[12:13]
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: image_bvh8_intersect_ray v[0:9], [v[24:25], v[26:27], v[21:23], v[18:20], v28], s[0:3]
+; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr10_vgpr11_vgpr12_vgpr13
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr24_vgpr25
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr26_vgpr27
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr28
-; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-SDAG-NEXT: s_cbranch_execnz .LBB3_1
; GFX12-SDAG-NEXT: ; %bb.2:
; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll
index a07960f803250..b843d8394f228 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll
@@ -185,11 +185,11 @@ define amdgpu_ps <10 x float> @image_bvh_dual_intersect_ray_vvvvvv(i64 %node_ptr
; GFX12-SDAG-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[13:14]
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: image_bvh_dual_intersect_ray v[0:9], [v[27:28], v[29:30], v[22:24], v[19:21], v[25:26]], s[0:3]
+; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr11_vgpr12_vgpr13_vgpr14
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr27_vgpr28
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr29_vgpr30
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr25_vgpr26
-; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-SDAG-NEXT: s_cbranch_execnz .LBB3_1
; GFX12-SDAG-NEXT: ; %bb.2:
; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
index 82c3874fe6b0d..de35c48f59e03 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
@@ -120,10 +120,10 @@ define <2 x bfloat> @raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v6
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v5, s[0:3], s6 offen offset:128 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr6
; GFX1200-NEXT: ; implicit-def: $vgpr5
-; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: s_cbranch_execnz .LBB4_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -154,11 +154,11 @@ define <2 x bfloat> @raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__
; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v6
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], s6 offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9_vgpr10_vgpr11
; GFX1250-NEXT: ; implicit-def: $vgpr6
; GFX1250-NEXT: ; implicit-def: $vgpr1
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: s_cbranch_execnz .LBB4_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
index c65422bc86810..574b45472f9f1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
@@ -23,9 +23,9 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX10-NEXT: buffer_load_format_d16_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX10-NEXT: s_andn2_wrexec_b32 s5, s5
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
-; GFX10-NEXT: s_andn2_wrexec_b32 s5, s5
; GFX10-NEXT: s_cbranch_execnz .LBB0_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -104,9 +104,9 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-NEXT: ; implicit-def: $vgpr4
-; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s4
@@ -131,9 +131,9 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], null idxen
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
-; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: s_cbranch_execnz .LBB0_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
index 6c8756b7e5f2f..1dca37a2f8fcc 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
@@ -56,10 +56,10 @@ define <2 x bfloat> @struct_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsr
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v[5:6], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: s_cbranch_execnz .LBB2_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -93,11 +93,11 @@ define void @struct_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__vgpr_rsrc__vgp
; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v[5:6], s[0:3], s6 idxen offen
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr0
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: s_cbranch_execnz .LBB3_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
index e6517afddc410..2a631b543f5aa 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
@@ -311,11 +311,11 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: buffer_atomic_add_f32 v0, v[5:6], s[0:3], s6 idxen offen
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr0
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: s_cbranch_execnz .LBB4_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -345,12 +345,12 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[0:3], s6 idxen offen
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr0
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: s_cbranch_execnz .LBB4_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s4
@@ -481,11 +481,11 @@ define void @struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr
; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: buffer_atomic_pk_add_f16 v0, v[5:6], s[0:3], s6 idxen offen
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr0
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: s_cbranch_execnz .LBB5_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -515,12 +515,12 @@ define void @struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[0:3], s6 idxen offen
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr0
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: s_cbranch_execnz .LBB5_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
index 729b2126a88e5..c3e88997d0ba3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
@@ -261,10 +261,10 @@ define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_add_f32 v0, v[5:6], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: s_cbranch_execnz .LBB4_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -296,11 +296,11 @@ define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: s_cbranch_execnz .LBB4_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s4
@@ -403,10 +403,10 @@ define <2 x half> @struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_pk_add_f16 v0, v[5:6], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: s_cbranch_execnz .LBB5_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -438,11 +438,11 @@ define <2 x half> @struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__
; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: s_cbranch_execnz .LBB5_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
index b40d866802f9b..f4d364eaa4458 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
@@ -473,9 +473,9 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmax v0, v[5:6], s[4:7], s16 idxen offen offset:256 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX10-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB8_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -499,9 +499,9 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_max_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX11-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX11-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX11-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX11-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s1
@@ -529,9 +529,9 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_max_num_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX1200-NEXT: s_cbranch_execnz .LBB8_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s1
@@ -560,10 +560,10 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[4:5]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_max_num_f32 v0, v[8:9], s[4:7], s0 idxen offen th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
-; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_and_not1_wrexec_b32 s2, s2
+; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
; GFX1250-NEXT: s_cbranch_execnz .LBB8_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s1
@@ -649,10 +649,10 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX10-NEXT: v_cmpx_eq_u32_e32 s10, v7
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmax v0, v[5:6], s[4:7], s10 idxen offen offset:256 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX10-NEXT: ; implicit-def: $vgpr7
; GFX10-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB9_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -679,10 +679,10 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX11-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_max_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX11-NEXT: ; implicit-def: $vgpr7
; GFX11-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s4
@@ -712,10 +712,10 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_max_num_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: s_cbranch_execnz .LBB9_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -746,11 +746,11 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_max_num_f32 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: s_cbranch_execnz .LBB9_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
index edabefdfdaf8d..d284c476c9333 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
@@ -473,9 +473,9 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmin v0, v[5:6], s[4:7], s16 idxen offen offset:256 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX10-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB8_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -499,9 +499,9 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_min_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX11-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX11-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX11-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX11-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s1
@@ -529,9 +529,9 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_min_num_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX1200-NEXT: s_cbranch_execnz .LBB8_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s1
@@ -560,10 +560,10 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[4:5]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_min_num_f32 v0, v[8:9], s[4:7], s0 idxen offen th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
-; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_and_not1_wrexec_b32 s2, s2
+; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
; GFX1250-NEXT: s_cbranch_execnz .LBB8_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s1
@@ -649,10 +649,10 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX10-NEXT: v_cmpx_eq_u32_e32 s10, v7
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmin v0, v[5:6], s[4:7], s10 idxen offen offset:256 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX10-NEXT: ; implicit-def: $vgpr7
; GFX10-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB9_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -679,10 +679,10 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX11-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_min_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX11-NEXT: ; implicit-def: $vgpr7
; GFX11-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s4
@@ -712,10 +712,10 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_min_num_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: s_cbranch_execnz .LBB9_1
; GFX1200-NEXT: ; %bb.2:
; GFX1200-NEXT: s_mov_b32 exec_lo, s4
@@ -746,11 +746,11 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_min_num_f32 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: s_cbranch_execnz .LBB9_1
; GFX1250-NEXT: ; %bb.2:
; GFX1250-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
index e0752819e7145..1db4db27e8841 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
@@ -20,9 +20,9 @@ define amdgpu_gs void @main(ptr addrspace(8) %arg, i32 %arg1) {
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX10-NEXT: buffer_load_format_d16_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX10-NEXT: s_andn2_wrexec_b32 s5, s5
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
-; GFX10-NEXT: s_andn2_wrexec_b32 s5, s5
; GFX10-NEXT: s_cbranch_execnz .LBB0_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -101,9 +101,9 @@ define amdgpu_gs void @main(ptr addrspace(8) %arg, i32 %arg1) {
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-NEXT: ; implicit-def: $vgpr4
-; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/mixed-vmem-types.ll b/llvm/test/CodeGen/AMDGPU/mixed-vmem-types.ll
index 15ffb3a7e7dcf..7fc67b832a9d8 100644
--- a/llvm/test/CodeGen/AMDGPU/mixed-vmem-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/mixed-vmem-types.ll
@@ -200,13 +200,13 @@ define amdgpu_cs void @mixed_vmem_types_bvh_reads(i32 inreg %descTable0, i32 inr
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[16:17], v[12:13]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[18:19], v[14:15]
; GFX11-NEXT: image_bvh64_intersect_ray v[15:18], v[0:11], s[16:19]
+; GFX11-NEXT: s_and_not1_wrexec_b32 s21, s21
; GFX11-NEXT: ; implicit-def: $vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX11-NEXT: ; implicit-def: $vgpr2
; GFX11-NEXT: ; implicit-def: $vgpr3_vgpr4_vgpr5
; GFX11-NEXT: ; implicit-def: $vgpr6_vgpr7_vgpr8
; GFX11-NEXT: ; implicit-def: $vgpr9_vgpr10_vgpr11
-; GFX11-NEXT: s_and_not1_wrexec_b32 s21, s21
; GFX11-NEXT: s_cbranch_execnz .LBB1_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s20
@@ -257,13 +257,13 @@ define amdgpu_cs void @mixed_vmem_types_bvh_reads(i32 inreg %descTable0, i32 inr
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[18:19], v[14:15]
; GFX12-NEXT: image_bvh64_intersect_ray v[15:18], [v[0:1], v2, v[3:5], v[6:8], v[9:11]], s[16:19]
+; GFX12-NEXT: s_and_not1_wrexec_b32 s21, s21
; GFX12-NEXT: ; implicit-def: $vgpr12_vgpr13_vgpr14_vgpr15
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT: ; implicit-def: $vgpr2
; GFX12-NEXT: ; implicit-def: $vgpr3_vgpr4_vgpr5
; GFX12-NEXT: ; implicit-def: $vgpr6_vgpr7_vgpr8
; GFX12-NEXT: ; implicit-def: $vgpr9_vgpr10_vgpr11
-; GFX12-NEXT: s_and_not1_wrexec_b32 s21, s21
; GFX12-NEXT: s_cbranch_execnz .LBB1_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s20
diff --git a/llvm/test/CodeGen/AMDGPU/move-to-valu-vimage-vsample.ll b/llvm/test/CodeGen/AMDGPU/move-to-valu-vimage-vsample.ll
index 289333f1d41e7..fb9c7ab876272 100644
--- a/llvm/test/CodeGen/AMDGPU/move-to-valu-vimage-vsample.ll
+++ b/llvm/test/CodeGen/AMDGPU/move-to-valu-vimage-vsample.ll
@@ -47,7 +47,8 @@ define amdgpu_ps float @vimage_move_to_valu(<8 x i32> %rsrc) {
; GFX11-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[IMAGE_LOAD_V1_V2_nsa_gfx11_:%[0-9]+]]:vgpr_32 = IMAGE_LOAD_V1_V2_nsa_gfx11 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE5]], 1, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX11-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX11-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX11-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX11-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.3:
@@ -99,7 +100,8 @@ define amdgpu_ps float @vimage_move_to_valu(<8 x i32> %rsrc) {
; GFX12-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[IMAGE_LOAD_V1_V2_gfx12_:%[0-9]+]]:vgpr_32 = IMAGE_LOAD_V1_V2_gfx12 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE5]], 1, 1, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX12-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX12-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX12-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
@@ -161,7 +163,8 @@ define amdgpu_ps float @vsample_move_to_valu_rsrc(<8 x i32> %rsrc, <4 x i32> inr
; GFX11-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[IMAGE_SAMPLE_V1_V1_gfx11_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V1_gfx11 [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE6]], [[REG_SEQUENCE1]], 1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX11-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX11-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX11-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX11-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.3:
@@ -218,7 +221,8 @@ define amdgpu_ps float @vsample_move_to_valu_rsrc(<8 x i32> %rsrc, <4 x i32> inr
; GFX12-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[IMAGE_SAMPLE_V1_V1_gfx12_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V1_gfx12 [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE6]], [[REG_SEQUENCE1]], 1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX12-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX12-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX12-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
@@ -272,7 +276,8 @@ define amdgpu_ps float @vsample_move_to_valu_samp(<8 x i32> inreg %rsrc, <4 x i3
; GFX11-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[IMAGE_SAMPLE_V1_V1_gfx11_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V1_gfx11 [[V_MOV_B32_e32_]], [[REG_SEQUENCE]], killed [[REG_SEQUENCE4]], 1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX11-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX11-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX11-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX11-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.3:
@@ -321,7 +326,8 @@ define amdgpu_ps float @vsample_move_to_valu_samp(<8 x i32> inreg %rsrc, <4 x i3
; GFX12-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[IMAGE_SAMPLE_V1_V1_gfx12_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V1_gfx12 [[V_MOV_B32_e32_]], [[REG_SEQUENCE]], killed [[REG_SEQUENCE4]], 1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX12-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX12-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; GFX12-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
index 7163c4607b8b4..610d80e9bba09 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
@@ -49,9 +49,9 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W32-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4
-; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB0_1
; GFX1010_W32-NEXT: ; %bb.2:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -74,9 +74,9 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W64-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4
-; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB0_1
; GFX1010_W64-NEXT: ; %bb.2:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -100,9 +100,9 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W32-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4
-; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB0_1
; GFX1100_W32-NEXT: ; %bb.2:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
@@ -125,9 +125,9 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W64-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4
-; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB0_1
; GFX1100_W64-NEXT: ; %bb.2:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
@@ -302,8 +302,8 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W32-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB1_1
; GFX1010_W32-NEXT: ; %bb.2:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -319,9 +319,9 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; GFX1010_W32-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W32-NEXT: ; implicit-def: $vgpr8
-; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB1_3
; GFX1010_W32-NEXT: ; %bb.4:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -348,8 +348,8 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W64-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB1_1
; GFX1010_W64-NEXT: ; %bb.2:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -365,9 +365,9 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; GFX1010_W64-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W64-NEXT: ; implicit-def: $vgpr8
-; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB1_3
; GFX1010_W64-NEXT: ; %bb.4:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -395,8 +395,8 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W32-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB1_1
; GFX1100_W32-NEXT: ; %bb.2:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
@@ -412,9 +412,9 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
; GFX1100_W32-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W32-NEXT: ; implicit-def: $vgpr8
-; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB1_3
; GFX1100_W32-NEXT: ; %bb.4:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
@@ -441,8 +441,8 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W64-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB1_1
; GFX1100_W64-NEXT: ; %bb.2:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
@@ -458,9 +458,9 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
; GFX1100_W64-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W64-NEXT: ; implicit-def: $vgpr8
-; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB1_3
; GFX1100_W64-NEXT: ; %bb.4:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
@@ -759,7 +759,7 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W32-NEXT: ;;#ASMSTART
; GFX1010_W32-NEXT: s_mov_b32 s4, 17
; GFX1010_W32-NEXT: ;;#ASMEND
-; GFX1010_W32-NEXT: v_mov_b32_e32 v8, s4
+; GFX1010_W32-NEXT: v_mov_b32_e32 v9, s4
; GFX1010_W32-NEXT: s_mov_b32 s5, exec_lo
; GFX1010_W32-NEXT: s_mov_b32 s6, s5
; GFX1010_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
@@ -770,10 +770,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[8:9], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[10:11], v[2:3]
-; GFX1010_W32-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr8
+; GFX1010_W32-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s6, s6
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr9
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB2_1
; GFX1010_W32-NEXT: ; %bb.2:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -794,10 +794,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
-; GFX1010_W32-NEXT: buffer_load_format_x v9, v0, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0
-; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB2_4
; GFX1010_W32-NEXT: ; %bb.5:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -805,7 +805,7 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W32-NEXT: .LBB2_6: ; %bb2
; GFX1010_W32-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W32-NEXT: global_store_dword v[11:12], v9, off
+; GFX1010_W32-NEXT: global_store_dword v[11:12], v8, off
; GFX1010_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W32-NEXT: s_setpc_b64 s[30:31]
;
@@ -815,7 +815,7 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W64-NEXT: ;;#ASMSTART
; GFX1010_W64-NEXT: s_mov_b32 s4, 17
; GFX1010_W64-NEXT: ;;#ASMEND
-; GFX1010_W64-NEXT: v_mov_b32_e32 v8, s4
+; GFX1010_W64-NEXT: v_mov_b32_e32 v9, s4
; GFX1010_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX1010_W64-NEXT: s_mov_b64 s[12:13], s[6:7]
; GFX1010_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
@@ -826,10 +826,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[8:9], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[10:11], v[2:3]
-; GFX1010_W64-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr8
+; GFX1010_W64-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr9
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB2_1
; GFX1010_W64-NEXT: ; %bb.2:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -850,10 +850,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
-; GFX1010_W64-NEXT: buffer_load_format_x v9, v0, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0
-; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB2_4
; GFX1010_W64-NEXT: ; %bb.5:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -861,7 +861,7 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W64-NEXT: .LBB2_6: ; %bb2
; GFX1010_W64-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W64-NEXT: global_store_dword v[11:12], v9, off
+; GFX1010_W64-NEXT: global_store_dword v[11:12], v8, off
; GFX1010_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W64-NEXT: s_setpc_b64 s[30:31]
;
@@ -871,7 +871,7 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W32-NEXT: ;;#ASMSTART
; GFX1100_W32-NEXT: s_mov_b32 s4, 17
; GFX1100_W32-NEXT: ;;#ASMEND
-; GFX1100_W32-NEXT: v_mov_b32_e32 v8, s4
+; GFX1100_W32-NEXT: v_mov_b32_e32 v9, s4
; GFX1100_W32-NEXT: s_mov_b32 s5, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W32-NEXT: s_mov_b32 s6, s5
@@ -883,10 +883,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX1100_W32-NEXT: buffer_load_format_x v9, v8, s[0:3], 0 idxen
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr8
+; GFX1100_W32-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr9
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB2_1
; GFX1100_W32-NEXT: ; %bb.2:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s5
@@ -908,10 +908,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
-; GFX1100_W32-NEXT: buffer_load_format_x v9, v0, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0
-; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB2_4
; GFX1100_W32-NEXT: ; %bb.5:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
@@ -919,7 +919,7 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W32-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W32-NEXT: global_store_b32 v[11:12], v9, off dlc
+; GFX1100_W32-NEXT: global_store_b32 v[11:12], v8, off dlc
; GFX1100_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W32-NEXT: s_setpc_b64 s[30:31]
;
@@ -929,7 +929,7 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W64-NEXT: ;;#ASMSTART
; GFX1100_W64-NEXT: s_mov_b32 s4, 17
; GFX1100_W64-NEXT: ;;#ASMEND
-; GFX1100_W64-NEXT: v_mov_b32_e32 v8, s4
+; GFX1100_W64-NEXT: v_mov_b32_e32 v9, s4
; GFX1100_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W64-NEXT: s_mov_b64 s[8:9], s[6:7]
@@ -941,10 +941,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX1100_W64-NEXT: buffer_load_format_x v9, v8, s[0:3], 0 idxen
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr8
+; GFX1100_W64-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr9
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB2_1
; GFX1100_W64-NEXT: ; %bb.2:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[6:7]
@@ -966,10 +966,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
-; GFX1100_W64-NEXT: buffer_load_format_x v9, v0, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0
-; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB2_4
; GFX1100_W64-NEXT: ; %bb.5:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
@@ -977,7 +977,7 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W64-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W64-NEXT: global_store_b32 v[11:12], v9, off dlc
+; GFX1100_W64-NEXT: global_store_b32 v[11:12], v8, off dlc
; GFX1100_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W64-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
index 6894f2ac9cf0e..6911408544888 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
@@ -48,9 +48,9 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W32-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4
-; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB0_1
; GFX1010_W32-NEXT: ; %bb.2:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -73,9 +73,9 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W64-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4
-; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB0_1
; GFX1010_W64-NEXT: ; %bb.2:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -99,9 +99,9 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W32-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4
-; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB0_1
; GFX1100_W32-NEXT: ; %bb.2:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
@@ -124,9 +124,9 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W64-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4
-; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB0_1
; GFX1100_W64-NEXT: ; %bb.2:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
@@ -308,8 +308,8 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W32-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB1_1
; GFX1010_W32-NEXT: ; %bb.2:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -325,9 +325,9 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; GFX1010_W32-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W32-NEXT: ; implicit-def: $vgpr8
-; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB1_3
; GFX1010_W32-NEXT: ; %bb.4:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -354,8 +354,8 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W64-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB1_1
; GFX1010_W64-NEXT: ; %bb.2:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -371,9 +371,9 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; GFX1010_W64-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W64-NEXT: ; implicit-def: $vgpr8
-; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB1_3
; GFX1010_W64-NEXT: ; %bb.4:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -401,8 +401,8 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W32-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB1_1
; GFX1100_W32-NEXT: ; %bb.2:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
@@ -418,9 +418,9 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
; GFX1100_W32-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W32-NEXT: ; implicit-def: $vgpr8
-; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB1_3
; GFX1100_W32-NEXT: ; %bb.4:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
@@ -447,8 +447,8 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W64-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB1_1
; GFX1100_W64-NEXT: ; %bb.2:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
@@ -464,9 +464,9 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
; GFX1100_W64-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W64-NEXT: ; implicit-def: $vgpr8
-; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB1_3
; GFX1100_W64-NEXT: ; %bb.4:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
@@ -779,7 +779,7 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W32-NEXT: ;;#ASMSTART
; GFX1010_W32-NEXT: s_mov_b32 s4, 17
; GFX1010_W32-NEXT: ;;#ASMEND
-; GFX1010_W32-NEXT: v_mov_b32_e32 v8, s4
+; GFX1010_W32-NEXT: v_mov_b32_e32 v9, s4
; GFX1010_W32-NEXT: s_mov_b32 s5, exec_lo
; GFX1010_W32-NEXT: s_mov_b32 s6, s5
; GFX1010_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
@@ -790,10 +790,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[8:9], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[10:11], v[2:3]
-; GFX1010_W32-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr8
+; GFX1010_W32-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s6, s6
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr9
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB2_1
; GFX1010_W32-NEXT: ; %bb.2:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -814,10 +814,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
-; GFX1010_W32-NEXT: buffer_load_format_x v9, v0, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0
-; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB2_4
; GFX1010_W32-NEXT: ; %bb.5:
; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -825,7 +825,7 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W32-NEXT: .LBB2_6: ; %bb2
; GFX1010_W32-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W32-NEXT: global_store_dword v[11:12], v9, off
+; GFX1010_W32-NEXT: global_store_dword v[11:12], v8, off
; GFX1010_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W32-NEXT: s_setpc_b64 s[30:31]
;
@@ -835,7 +835,7 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W64-NEXT: ;;#ASMSTART
; GFX1010_W64-NEXT: s_mov_b32 s4, 17
; GFX1010_W64-NEXT: ;;#ASMEND
-; GFX1010_W64-NEXT: v_mov_b32_e32 v8, s4
+; GFX1010_W64-NEXT: v_mov_b32_e32 v9, s4
; GFX1010_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX1010_W64-NEXT: s_mov_b64 s[12:13], s[6:7]
; GFX1010_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
@@ -846,10 +846,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[8:9], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[10:11], v[2:3]
-; GFX1010_W64-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr8
+; GFX1010_W64-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr9
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB2_1
; GFX1010_W64-NEXT: ; %bb.2:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -870,10 +870,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
-; GFX1010_W64-NEXT: buffer_load_format_x v9, v0, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0
-; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB2_4
; GFX1010_W64-NEXT: ; %bb.5:
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -881,7 +881,7 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W64-NEXT: .LBB2_6: ; %bb2
; GFX1010_W64-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W64-NEXT: global_store_dword v[11:12], v9, off
+; GFX1010_W64-NEXT: global_store_dword v[11:12], v8, off
; GFX1010_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W64-NEXT: s_setpc_b64 s[30:31]
;
@@ -891,7 +891,7 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W32-NEXT: ;;#ASMSTART
; GFX1100_W32-NEXT: s_mov_b32 s4, 17
; GFX1100_W32-NEXT: ;;#ASMEND
-; GFX1100_W32-NEXT: v_mov_b32_e32 v8, s4
+; GFX1100_W32-NEXT: v_mov_b32_e32 v9, s4
; GFX1100_W32-NEXT: s_mov_b32 s5, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W32-NEXT: s_mov_b32 s6, s5
@@ -903,10 +903,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX1100_W32-NEXT: buffer_load_format_x v9, v8, s[0:3], 0 idxen
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr8
+; GFX1100_W32-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr9
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB2_1
; GFX1100_W32-NEXT: ; %bb.2:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s5
@@ -928,10 +928,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
-; GFX1100_W32-NEXT: buffer_load_format_x v9, v0, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0
-; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB2_4
; GFX1100_W32-NEXT: ; %bb.5:
; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
@@ -939,7 +939,7 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W32-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W32-NEXT: global_store_b32 v[11:12], v9, off dlc
+; GFX1100_W32-NEXT: global_store_b32 v[11:12], v8, off dlc
; GFX1100_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W32-NEXT: s_setpc_b64 s[30:31]
;
@@ -949,7 +949,7 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W64-NEXT: ;;#ASMSTART
; GFX1100_W64-NEXT: s_mov_b32 s4, 17
; GFX1100_W64-NEXT: ;;#ASMEND
-; GFX1100_W64-NEXT: v_mov_b32_e32 v8, s4
+; GFX1100_W64-NEXT: v_mov_b32_e32 v9, s4
; GFX1100_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W64-NEXT: s_mov_b64 s[8:9], s[6:7]
@@ -961,10 +961,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX1100_W64-NEXT: buffer_load_format_x v9, v8, s[0:3], 0 idxen
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr8
+; GFX1100_W64-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr9
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB2_1
; GFX1100_W64-NEXT: ; %bb.2:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[6:7]
@@ -986,10 +986,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
-; GFX1100_W64-NEXT: buffer_load_format_x v9, v0, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0
-; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB2_4
; GFX1100_W64-NEXT: ; %bb.5:
; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
@@ -997,7 +997,7 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1100_W64-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W64-NEXT: global_store_b32 v[11:12], v9, off dlc
+; GFX1100_W64-NEXT: global_store_b32 v[11:12], v8, off dlc
; GFX1100_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W64-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir
index 29ab3c8895af3..0a30df52b6f8e 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir
@@ -101,7 +101,8 @@ body: |
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W64-NEXT: {{ $}}
; W64-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W64-NEXT: [[S_ANDN2_WREXEC_B64_term:%[0-9]+]]:sreg_64_xexec = S_ANDN2_WREXEC_B64_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W64-NEXT: [[S_ANDN2_B64_:%[0-9]+]]:sreg_64_xexec = S_ANDN2_B64 [[PHI]], $exec, implicit-def $scc
+ ; W64-NEXT: $exec = S_MOV_B64_term [[S_ANDN2_B64_]]
; W64-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .3:
@@ -142,7 +143,8 @@ body: |
; W32-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W32-NEXT: {{ $}}
; W32-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W32-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W32-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; W32-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; W32-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .3:
@@ -251,7 +253,8 @@ body: |
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W64-NEXT: {{ $}}
; W64-NEXT: [[BUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W64-NEXT: [[S_ANDN2_WREXEC_B64_term:%[0-9]+]]:sreg_64_xexec = S_ANDN2_WREXEC_B64_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W64-NEXT: [[S_ANDN2_B64_:%[0-9]+]]:sreg_64_xexec = S_ANDN2_B64 [[PHI]], $exec, implicit-def $scc
+ ; W64-NEXT: $exec = S_MOV_B64_term [[S_ANDN2_B64_]]
; W64-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .3:
@@ -292,7 +295,8 @@ body: |
; W32-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W32-NEXT: {{ $}}
; W32-NEXT: [[BUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W32-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W32-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; W32-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; W32-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .3:
@@ -402,7 +406,8 @@ body: |
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W64-NEXT: {{ $}}
; W64-NEXT: [[BUFFER_LOAD_FORMAT_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_BOTHEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W64-NEXT: [[S_ANDN2_WREXEC_B64_term:%[0-9]+]]:sreg_64_xexec = S_ANDN2_WREXEC_B64_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W64-NEXT: [[S_ANDN2_B64_:%[0-9]+]]:sreg_64_xexec = S_ANDN2_B64 [[PHI]], $exec, implicit-def $scc
+ ; W64-NEXT: $exec = S_MOV_B64_term [[S_ANDN2_B64_]]
; W64-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .3:
@@ -443,7 +448,8 @@ body: |
; W32-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W32-NEXT: {{ $}}
; W32-NEXT: [[BUFFER_LOAD_FORMAT_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_BOTHEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W32-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W32-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; W32-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; W32-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .3:
@@ -671,7 +677,8 @@ body: |
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W64-NEXT: {{ $}}
; W64-NEXT: [[BUFFER_LOAD_FORMAT_X_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFSET killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W64-NEXT: [[S_ANDN2_WREXEC_B64_term:%[0-9]+]]:sreg_64_xexec = S_ANDN2_WREXEC_B64_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W64-NEXT: [[S_ANDN2_B64_:%[0-9]+]]:sreg_64_xexec = S_ANDN2_B64 [[PHI]], $exec, implicit-def $scc
+ ; W64-NEXT: $exec = S_MOV_B64_term [[S_ANDN2_B64_]]
; W64-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .3:
@@ -712,7 +719,8 @@ body: |
; W32-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W32-NEXT: {{ $}}
; W32-NEXT: [[BUFFER_LOAD_FORMAT_X_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFSET killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W32-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W32-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def $scc
+ ; W32-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; W32-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .3:
diff --git a/llvm/test/CodeGen/AMDGPU/si-optimize-exec-masking-andn2-wrexec.mir b/llvm/test/CodeGen/AMDGPU/si-optimize-exec-masking-andn2-wrexec.mir
new file mode 100644
index 0000000000000..dfa62197990b7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/si-optimize-exec-masking-andn2-wrexec.mir
@@ -0,0 +1,140 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+wavefrontsize32 --run-pass=si-optimize-exec-masking -o - %s | FileCheck %s
+
+# SIOptimizeExecMasking folds the split EXEC update emitted by the waterfall
+# loop back into S_ANDN2_WREXEC.
+
+---
+name: fold_adjacent
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: fold_adjacent
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.0(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: liveins: $sgpr4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $sgpr4 = S_ANDN2_WREXEC_B32 $sgpr4, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_EXECNZ %bb.0, implicit $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ liveins: $sgpr4
+ $sgpr4 = S_ANDN2_B32 $sgpr4, $exec_lo, implicit-def dead $scc
+ $exec_lo = S_MOV_B32_term $sgpr4
+ S_CBRANCH_EXECNZ %bb.0, implicit $exec
+
+ bb.1:
+ S_ENDPGM 0
+...
+
+---
+# The register allocator may leave meta instructions between the pair.
+name: fold_across_meta
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: fold_across_meta
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.0(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: liveins: $sgpr4, $vgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $sgpr4 = S_ANDN2_WREXEC_B32 $sgpr4, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: KILL implicit-def $vgpr0
+ ; CHECK-NEXT: S_CBRANCH_EXECNZ %bb.0, implicit $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ liveins: $sgpr4, $vgpr0
+ $sgpr4 = S_ANDN2_B32 $sgpr4, $exec_lo, implicit-def dead $scc
+ KILL implicit-def $vgpr0
+ $exec_lo = S_MOV_B32_term $sgpr4
+ S_CBRANCH_EXECNZ %bb.0, implicit $exec
+
+ bb.1:
+ S_ENDPGM 0
+...
+
+---
+# The scheduler may separate the pair by real instructions.
+name: fold_across_instr
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: fold_across_instr
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.0(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: liveins: $sgpr4, $sgpr6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $sgpr6 = S_ADD_I32 $sgpr6, 1, implicit-def dead $scc
+ ; CHECK-NEXT: $sgpr4 = S_ANDN2_WREXEC_B32 $sgpr4, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_EXECNZ %bb.0, implicit $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ liveins: $sgpr4, $sgpr6
+ $sgpr4 = S_ANDN2_B32 $sgpr4, $exec_lo, implicit-def dead $scc
+ $sgpr6 = S_ADD_I32 $sgpr6, 1, implicit-def dead $scc
+ $exec_lo = S_MOV_B32_term $sgpr4
+ S_CBRANCH_EXECNZ %bb.0, implicit $exec
+
+ bb.1:
+ S_ENDPGM 0
+...
+
+---
+# S_ANDN2_WREXEC clobbers SCC where the EXEC write is, so do not fold when SCC
+# is live out.
+name: no_fold_scc_live
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: no_fold_scc_live
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.0(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: liveins: $sgpr4, $sgpr6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $sgpr4 = S_ANDN2_B32 $sgpr4, $exec_lo, implicit-def dead $scc
+ ; CHECK-NEXT: $sgpr6 = S_ADD_I32 $sgpr6, 1, implicit-def $scc
+ ; CHECK-NEXT: $exec_lo = COPY $sgpr4
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.0, implicit $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ liveins: $sgpr4, $sgpr6
+ $sgpr4 = S_ANDN2_B32 $sgpr4, $exec_lo, implicit-def dead $scc
+ $sgpr6 = S_ADD_I32 $sgpr6, 1, implicit-def $scc
+ $exec_lo = S_MOV_B32_term $sgpr4
+ S_CBRANCH_SCC1 %bb.0, implicit $scc
+
+ bb.1:
+ S_ENDPGM 0
+...
+
+---
+# The source of the S_ANDN2 must not be redefined before the EXEC write.
+name: no_fold_src_clobbered
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: no_fold_src_clobbered
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.0(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: liveins: $sgpr4, $sgpr5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $sgpr4 = S_ANDN2_B32 $sgpr5, $exec_lo, implicit-def dead $scc
+ ; CHECK-NEXT: $sgpr5 = S_MOV_B32 0
+ ; CHECK-NEXT: $exec_lo = COPY $sgpr4
+ ; CHECK-NEXT: S_CBRANCH_EXECNZ %bb.0, implicit $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ liveins: $sgpr4, $sgpr5
+ $sgpr4 = S_ANDN2_B32 $sgpr5, $exec_lo, implicit-def dead $scc
+ $sgpr5 = S_MOV_B32 0
+ $exec_lo = S_MOV_B32_term $sgpr4
+ S_CBRANCH_EXECNZ %bb.0, implicit $exec
+
+ bb.1:
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll b/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
index ba0024ff16dea..960b6e61f7c83 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
@@ -27,8 +27,8 @@ define void @vgpr_descriptor_waterfall_loop_idom_update(ptr %arg) #0 {
; GCN-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[2:3]
; GCN-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[4:5]
; GCN-NEXT: buffer_store_dword v0, v0, s[4:7], 0 offen
-; GCN-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GCN-NEXT: s_andn2_wrexec_b32 s9, s9
+; GCN-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GCN-NEXT: s_cbranch_execnz .LBB0_2
; GCN-NEXT: ; %bb.3: ; in Loop: Header=BB0_1 Depth=1
; GCN-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -60,8 +60,8 @@ define void @vgpr_descriptor_waterfall_loop_idom_update(ptr %arg) #0 {
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
; GFX11-NEXT: buffer_store_b32 v0, v0, s[0:3], 0 offen
-; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX11-NEXT: s_cbranch_execnz .LBB0_2
; GFX11-NEXT: ; %bb.3: ; in Loop: Header=BB0_1 Depth=1
; GFX11-NEXT: s_mov_b32 exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
index 577c8a28e42db..87116a71f6061 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
@@ -266,7 +266,8 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed [[REG_SEQUENCE1]], 0, csr_amdgpu_si_gfx, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed $vgpr0, implicit-def $vgpr0
; SI-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
- ; SI-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term killed [[PHI4]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; SI-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 killed [[PHI4]], $exec_lo, implicit-def dead $scc
+ ; SI-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; SI-NEXT: SI_WATERFALL_LOOP %bb.3, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.5:
@@ -303,7 +304,8 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed [[REG_SEQUENCE3]], 0, csr_amdgpu_si_gfx, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed $vgpr0, implicit-def $vgpr0
; SI-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
- ; SI-NEXT: [[S_ANDN2_WREXEC_B32_term1:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term killed [[PHI7]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; SI-NEXT: [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 killed [[PHI7]], $exec_lo, implicit-def dead $scc
+ ; SI-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_1]]
; SI-NEXT: SI_WATERFALL_LOOP %bb.7, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.9:
@@ -387,7 +389,8 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed [[REG_SEQUENCE1]], 0, csr_amdgpu_si_gfx, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed $vgpr0, implicit-def $vgpr0
; SI-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
- ; SI-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term killed [[PHI3]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; SI-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 killed [[PHI3]], $exec_lo, implicit-def dead $scc
+ ; SI-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; SI-NEXT: SI_WATERFALL_LOOP %bb.3, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.5:
@@ -423,7 +426,8 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed [[REG_SEQUENCE3]], 0, csr_amdgpu_si_gfx, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed $vgpr0, implicit-def $vgpr0
; SI-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
- ; SI-NEXT: [[S_ANDN2_WREXEC_B32_term1:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term killed [[PHI5]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; SI-NEXT: [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 killed [[PHI5]], $exec_lo, implicit-def dead $scc
+ ; SI-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_1]]
; SI-NEXT: SI_WATERFALL_LOOP %bb.7, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.9:
@@ -624,14 +628,16 @@ define protected amdgpu_kernel void @nested_waterfalls(ptr addrspace(1) %tex.coe
; SI-NEXT: successors: %bb.4(0x40000000), %bb.6(0x40000000)
; SI-NEXT: {{ $}}
; SI-NEXT: [[IMAGE_SAMPLE_V1_V2_nsa_gfx10_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V2_nsa_gfx10 undef %29:vgpr_32, undef %31:vgpr_32, [[REG_SEQUENCE5]], killed [[REG_SEQUENCE8]], 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; SI-NEXT: [[S_ANDN2_WREXEC_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term killed [[PHI1]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; SI-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 killed [[PHI1]], $exec_lo, implicit-def dead $scc
+ ; SI-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_]]
; SI-NEXT: SI_WATERFALL_LOOP %bb.4, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.6:
; SI-NEXT: successors: %bb.2(0x40000000), %bb.7(0x40000000)
; SI-NEXT: {{ $}}
; SI-NEXT: $exec_lo = S_MOV_B32 killed [[S_MOV_B32_1]]
- ; SI-NEXT: [[S_ANDN2_WREXEC_B32_term1:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term killed [[PHI]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; SI-NEXT: [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 killed [[PHI]], $exec_lo, implicit-def dead $scc
+ ; SI-NEXT: $exec_lo = S_MOV_B32_term [[S_ANDN2_B32_1]]
; SI-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.7:
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll b/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll
index ab9ae948d48de..cdec79cc385e4 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll
@@ -187,9 +187,9 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: s_mov_b64 s[2:3], s[14:15]
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
; SI-NEXT: v_mov_b32_e32 v1, v0
+; SI-NEXT: s_andn2_wrexec_b32 s8, s8
; SI-NEXT: ; implicit-def: $vgpr4_vgpr5
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: s_andn2_wrexec_b32 s8, s8
; SI-NEXT: s_cbranch_execnz .LBB3_2
; SI-NEXT: ; %bb.3:
; SI-NEXT: s_mov_b32 exec_lo, s7
@@ -210,9 +210,9 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: s_mov_b64 s[2:3], s[14:15]
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
; SI-NEXT: v_mov_b32_e32 v1, v0
+; SI-NEXT: s_andn2_wrexec_b32 s8, s8
; SI-NEXT: ; implicit-def: $vgpr2_vgpr3
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: s_andn2_wrexec_b32 s8, s8
; SI-NEXT: s_cbranch_execnz .LBB3_6
; SI-NEXT: ; %bb.7:
; SI-NEXT: s_mov_b32 exec_lo, s7
@@ -266,8 +266,8 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: s_mov_b64 s[0:1], s[12:13]
; SI-NEXT: s_mov_b64 s[2:3], s[14:15]
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
-; SI-NEXT: ; implicit-def: $vgpr4_vgpr5
; SI-NEXT: s_andn2_wrexec_b32 s8, s8
+; SI-NEXT: ; implicit-def: $vgpr4_vgpr5
; SI-NEXT: s_cbranch_execnz .LBB4_2
; SI-NEXT: ; %bb.3:
; SI-NEXT: s_mov_b32 exec_lo, s7
@@ -287,8 +287,8 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: s_mov_b64 s[0:1], s[12:13]
; SI-NEXT: s_mov_b64 s[2:3], s[14:15]
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
-; SI-NEXT: ; implicit-def: $vgpr2_vgpr3
; SI-NEXT: s_andn2_wrexec_b32 s8, s8
+; SI-NEXT: ; implicit-def: $vgpr2_vgpr3
; SI-NEXT: s_cbranch_execnz .LBB4_6
; SI-NEXT: ; %bb.7:
; SI-NEXT: s_mov_b32 exec_lo, s7
diff --git a/llvm/test/CodeGen/AMDGPU/waterfall-loop-wrexec-terminator.mir b/llvm/test/CodeGen/AMDGPU/waterfall-loop-exec-update-terminator.mir
similarity index 61%
rename from llvm/test/CodeGen/AMDGPU/waterfall-loop-wrexec-terminator.mir
rename to llvm/test/CodeGen/AMDGPU/waterfall-loop-exec-update-terminator.mir
index 817684257b274..cba7d41ea815f 100644
--- a/llvm/test/CodeGen/AMDGPU/waterfall-loop-wrexec-terminator.mir
+++ b/llvm/test/CodeGen/AMDGPU/waterfall-loop-exec-update-terminator.mir
@@ -1,15 +1,13 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgpu10.10 -mattr=+wavefrontsize32 -verify-machineinstrs --run-pass=si-fix-sgpr-copies,livevars,phi-node-elimination -o - %s | FileCheck %s --check-prefixes=W32
-# RUN: llc -mtriple=amdgpu10.10 -mattr=+wavefrontsize64 -verify-machineinstrs --run-pass=si-fix-sgpr-copies,livevars,phi-node-elimination -o - %s | FileCheck %s --check-prefixes=W64
-# RUN: llc -mtriple=amdgpu10.10 -mattr=+wavefrontsize32 -O0 -verify-machineinstrs --run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s --check-prefixes=O0
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+wavefrontsize32 --run-pass=si-fix-sgpr-copies,livevars,phi-node-elimination -o - %s | FileCheck %s --check-prefixes=W32
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+wavefrontsize64 --run-pass=si-fix-sgpr-copies,livevars,phi-node-elimination -o - %s | FileCheck %s --check-prefixes=W64
-# The waterfall loop EXEC update is a terminator, so PHI elimination has to
-# fold the exec PHI into its def instead of inserting a COPY after it. At -O0
-# the plain, non-terminator form is used because RegAllocFast would place the
-# spill of the def after the terminator.
+# The waterfall loop computes the new EXEC value into a virtual register and
+# writes EXEC from a terminator. PHI elimination therefore places the copy for
+# the loop-carried lane mask before the EXEC write, while EXEC is still full.
---
-name: waterfall_wrexec_term
+name: waterfall_exec_update
tracksRegLiveness: true
liveins:
- { reg: '$vgpr0', virtual-reg: '%0' }
@@ -20,7 +18,7 @@ liveins:
body: |
bb.0:
liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
- ; W32-LABEL: name: waterfall_wrexec_term
+ ; W32-LABEL: name: waterfall_exec_update
; W32: successors: %bb.1(0x80000000)
; W32-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
; W32-NEXT: {{ $}}
@@ -52,7 +50,9 @@ body: |
; W32-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W32-NEXT: {{ $}}
; W32-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[COPY]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W32-NEXT: [[COPY5:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32_term killed [[COPY6]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; W32-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_B32 killed [[COPY6]], $exec_lo, implicit-def dead $scc
+ ; W32-NEXT: [[COPY5:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[S_ANDN2_B32_]]
+ ; W32-NEXT: $exec_lo = S_MOV_B32_term killed [[S_ANDN2_B32_]]
; W32-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .3:
@@ -60,7 +60,7 @@ body: |
; W32-NEXT: $vgpr0 = COPY killed [[BUFFER_LOAD_FORMAT_X_IDXEN]]
; W32-NEXT: S_ENDPGM 0, implicit killed $vgpr0
;
- ; W64-LABEL: name: waterfall_wrexec_term
+ ; W64-LABEL: name: waterfall_exec_update
; W64: successors: %bb.1(0x80000000)
; W64-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
; W64-NEXT: {{ $}}
@@ -92,52 +92,15 @@ body: |
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W64-NEXT: {{ $}}
; W64-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[COPY]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W64-NEXT: [[COPY5:%[0-9]+]]:sreg_64_xexec = S_ANDN2_WREXEC_B64_term killed [[COPY6]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; W64-NEXT: [[S_ANDN2_B64_:%[0-9]+]]:sreg_64_xexec = S_ANDN2_B64 killed [[COPY6]], $exec, implicit-def dead $scc
+ ; W64-NEXT: [[COPY5:%[0-9]+]]:sreg_64_xexec = COPY [[S_ANDN2_B64_]]
+ ; W64-NEXT: $exec = S_MOV_B64_term killed [[S_ANDN2_B64_]]
; W64-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .3:
; W64-NEXT: $exec = S_MOV_B64 killed [[S_MOV_B64_]]
; W64-NEXT: $vgpr0 = COPY killed [[BUFFER_LOAD_FORMAT_X_IDXEN]]
; W64-NEXT: S_ENDPGM 0, implicit killed $vgpr0
- ;
- ; O0-LABEL: name: waterfall_wrexec_term
- ; O0: successors: %bb.1(0x80000000)
- ; O0-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
- ; O0-NEXT: {{ $}}
- ; O0-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr4
- ; O0-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; O0-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; O0-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; O0-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; O0-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
- ; O0-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
- ; O0-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
- ; O0-NEXT: {{ $}}
- ; O0-NEXT: .1:
- ; O0-NEXT: successors: %bb.2(0x80000000)
- ; O0-NEXT: {{ $}}
- ; O0-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %14, %bb.2
- ; O0-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
- ; O0-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
- ; O0-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; O0-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
- ; O0-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
- ; O0-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; O0-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; O0-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
- ; O0-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
- ; O0-NEXT: {{ $}}
- ; O0-NEXT: .2:
- ; O0-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
- ; O0-NEXT: {{ $}}
- ; O0-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[COPY]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; O0-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
- ; O0-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
- ; O0-NEXT: {{ $}}
- ; O0-NEXT: .3:
- ; O0-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
- ; O0-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_IDXEN]]
- ; O0-NEXT: S_ENDPGM 0, implicit $vgpr0
%4:vgpr_32 = COPY $vgpr4
%3:vgpr_32 = COPY $vgpr3
%2:vgpr_32 = COPY $vgpr2
>From 6f29cd6ec069398b430b07af9fd2d21f3679afc9 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Sun, 13 Sep 2026 21:03:06 +0200
Subject: [PATCH 3/3] address comments
---
.../Target/AMDGPU/SIOptimizeExecMasking.cpp | 33 +++++++++----------
.../si-optimize-exec-masking-andn2-wrexec.mir | 2 +-
2 files changed, 17 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
index 245122e141a9f..bc8a54616c966 100644
--- a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
+++ b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
@@ -64,6 +64,7 @@ class SIOptimizeExecMasking {
SmallVectorImpl<MachineOperand *> *KillFlagCandidates = nullptr,
unsigned MaxInstructions = 20) const;
bool optimizeExecSequence();
+ bool blocksAndN2Sink(const MachineInstr &MI, Register Dst) const;
bool optimizeAndN2WrExecSequence(MachineInstr &CopyToExecInst,
Register Dst) const;
void tryRecordVCmpxAndSaveexecSequence(MachineInstr &MI);
@@ -608,6 +609,20 @@ bool SIOptimizeExecMasking::optimizeExecSequence() {
return Changed;
}
+bool SIOptimizeExecMasking::blocksAndN2Sink(const MachineInstr &MI,
+ Register Dst) const {
+ for (const MachineOperand &MO : MI.operands()) {
+ if (!MO.isReg())
+ continue;
+ if (TRI->regsOverlap(MO.getReg(), Dst))
+ return true;
+ if (MO.isUse() ? TRI->regsOverlap(MO.getReg(), AMDGPU::SCC)
+ : TRI->regsOverlap(MO.getReg(), LMC.ExecReg))
+ return true;
+ }
+ return false;
+}
+
// Fold
//
// sdst = S_ANDN2_B32 ssrc, exec
@@ -648,23 +663,7 @@ bool SIOptimizeExecMasking::optimizeAndN2WrExecSequence(
AndN2Inst = &MI;
break;
}
- bool ReadsDst = false, ModifiesDst = false, ModifiesExec = false,
- ReadsSCC = false;
- for (const MachineOperand &MO : MI.operands()) {
- if (!MO.isReg())
- continue;
- if (TRI->regsOverlap(MO.getReg(), Dst)) {
- if (MO.isUse())
- ReadsDst = true;
- else
- ModifiesDst = true;
- }
- if (!MO.isUse() && TRI->regsOverlap(MO.getReg(), LMC.ExecReg))
- ModifiesExec = true;
- if (MO.isUse() && TRI->regsOverlap(MO.getReg(), AMDGPU::SCC))
- ReadsSCC = true;
- }
- if (ReadsDst || ModifiesDst || ModifiesExec || ReadsSCC)
+ if (blocksAndN2Sink(MI, Dst))
return false;
}
diff --git a/llvm/test/CodeGen/AMDGPU/si-optimize-exec-masking-andn2-wrexec.mir b/llvm/test/CodeGen/AMDGPU/si-optimize-exec-masking-andn2-wrexec.mir
index dfa62197990b7..7e287b0cacd1f 100644
--- a/llvm/test/CodeGen/AMDGPU/si-optimize-exec-masking-andn2-wrexec.mir
+++ b/llvm/test/CodeGen/AMDGPU/si-optimize-exec-masking-andn2-wrexec.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+wavefrontsize32 --run-pass=si-optimize-exec-masking -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu10.10-amd-amdhsa -run-pass=si-optimize-exec-masking -o - %s | FileCheck %s
# SIOptimizeExecMasking folds the split EXEC update emitted by the waterfall
# loop back into S_ANDN2_WREXEC.
More information about the llvm-commits
mailing list