[llvm] [AMDGPU] Model structured control-flow lane masks as i1 (PR #209158)
Pankaj Dwivedi via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 13 05:17:18 PDT 2026
https://github.com/PankajDwivedi-25 created https://github.com/llvm/llvm-project/pull/209158
Ref: #206435.
The lane mask from the structured CF intrinsics (`if`, `else`, `if.break`, `loop`, `end.cf`) is a per-lane value, but it was typed as an overloaded `i32`/`i64`.
This patch attempts to retype these intrinsics to `i1`, the natural type for a wave-width lane mask.
>From 63e2565b71c0aa3f9ef65bccde28a37b71ad7964 Mon Sep 17 00:00:00 2001
From: padivedi <pankajkumar.divedi at amd.com>
Date: Mon, 13 Jul 2026 16:43:55 +0530
Subject: [PATCH 1/2] [AMDGPU] Add pre-commit test demonstrating current
control-flow lowering (NFC)
---
.../AMDGPU/divergent-loop-i1-lane-mask.ll | 110 ++++++++++++++++++
.../AMDGPU/si-annotate-loop-i1-lane-mask.ll | 43 +++++++
2 files changed, 153 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/divergent-loop-i1-lane-mask.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/si-annotate-loop-i1-lane-mask.ll
diff --git a/llvm/test/CodeGen/AMDGPU/divergent-loop-i1-lane-mask.ll b/llvm/test/CodeGen/AMDGPU/divergent-loop-i1-lane-mask.ll
new file mode 100644
index 0000000000000..5864194a7c8a1
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/divergent-loop-i1-lane-mask.ll
@@ -0,0 +1,110 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9-SDAG %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10-SDAG %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10-GISEL %s
+
+; A divergent loop that the structurizer lowers using the control flow
+; intrinsics: llvm.amdgcn.if.break -> phi.broken -> llvm.amdgcn.end.cf. These
+; intrinsics now return the lane mask as an i1 instead of an i32/i64, which is
+; the natural type for a wave-width lane mask. Check that the mask is threaded
+; through the loop in a lane mask register on both the SelectionDAG and
+; GlobalISel paths.
+
+define amdgpu_kernel void @divergent_loop(ptr addrspace(1) %p) {
+; GFX9-SDAG-LABEL: divergent_loop:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX9-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-SDAG-NEXT: .LBB0_1: ; %loop
+; GFX9-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: s_add_i32 s2, s2, 1
+; GFX9-SDAG-NEXT: v_cmp_ge_i32_e32 vcc, s2, v0
+; GFX9-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB0_1
+; GFX9-SDAG-NEXT: ; %bb.2: ; %exit
+; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: divergent_loop:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX9-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-GISEL-NEXT: .LBB0_1: ; %loop
+; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-GISEL-NEXT: s_add_i32 s2, s2, 1
+; GFX9-GISEL-NEXT: v_cmp_ge_i32_e32 vcc, s2, v0
+; GFX9-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB0_1
+; GFX9-GISEL-NEXT: ; %bb.2: ; %exit
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
+;
+; GFX10-SDAG-LABEL: divergent_loop:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_mov_b32 s1, -1
+; GFX10-SDAG-NEXT: s_mov_b32 s0, 0
+; GFX10-SDAG-NEXT: .LBB0_1: ; %loop
+; GFX10-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-SDAG-NEXT: s_add_i32 s1, s1, 1
+; GFX10-SDAG-NEXT: v_cmp_ge_i32_e32 vcc_lo, s1, v0
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-SDAG-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX10-SDAG-NEXT: s_andn2_b32 exec_lo, exec_lo, s0
+; GFX10-SDAG-NEXT: s_cbranch_execnz .LBB0_1
+; GFX10-SDAG-NEXT: ; %bb.2: ; %exit
+; GFX10-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-SDAG-NEXT: s_endpgm
+;
+; GFX10-GISEL-LABEL: divergent_loop:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_mov_b32 s1, -1
+; GFX10-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX10-GISEL-NEXT: .LBB0_1: ; %loop
+; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-GISEL-NEXT: s_add_i32 s1, s1, 1
+; GFX10-GISEL-NEXT: v_cmp_ge_i32_e32 vcc_lo, s1, v0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s0
+; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB0_1
+; GFX10-GISEL-NEXT: ; %bb.2: ; %exit
+; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add i32 %i, 1
+ %cond = icmp slt i32 %i, %tid
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ %i.lcssa = phi i32 [ %i, %loop ]
+ store i32 %i.lcssa, ptr addrspace(1) %p
+ ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/si-annotate-loop-i1-lane-mask.ll b/llvm/test/CodeGen/AMDGPU/si-annotate-loop-i1-lane-mask.ll
new file mode 100644
index 0000000000000..90624e7cc9f33
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/si-annotate-loop-i1-lane-mask.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=amdgcn-- -S -structurizecfg -si-annotate-control-flow < %s | FileCheck -check-prefix=OPT %s
+
+; A divergent loop that the structurizer lowers using the control flow
+; intrinsics. Check the annotated IR: llvm.amdgcn.if.break, llvm.amdgcn.loop and
+; llvm.amdgcn.end.cf now thread the lane mask as an i1 (via the "phi.broken" phi)
+; instead of an i32/i64 wide integer.
+
+define amdgpu_kernel void @divergent_loop(ptr addrspace(1) %p) {
+; OPT-LABEL: @divergent_loop(
+; OPT-NEXT: entry:
+; OPT-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; OPT-NEXT: br label [[LOOP:%.*]]
+; OPT: loop:
+; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP0:%.*]], [[LOOP]] ], [ 0, [[ENTRY:%.*]] ]
+; OPT-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
+; OPT-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; OPT-NEXT: [[COND:%.*]] = icmp sge i32 [[I]], [[TID]]
+; OPT-NEXT: [[TMP0]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[COND]], i64 [[PHI_BROKEN]])
+; OPT-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP0]])
+; OPT-NEXT: br i1 [[TMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; OPT: exit:
+; OPT-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP0]])
+; OPT-NEXT: store i32 [[I]], ptr addrspace(1) [[P:%.*]], align 4
+; OPT-NEXT: ret void
+;
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add i32 %i, 1
+ %cond = icmp slt i32 %i, %tid
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ %i.lcssa = phi i32 [ %i, %loop ]
+ store i32 %i.lcssa, ptr addrspace(1) %p
+ ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x()
>From 5aaadee476bd750d25d2b45184e6285d79c3d5b9 Mon Sep 17 00:00:00 2001
From: padivedi <pankajkumar.divedi at amd.com>
Date: Mon, 13 Jul 2026 17:00:46 +0530
Subject: [PATCH 2/2] [AMDGPU] Model structured control-flow lane masks as i1
Co-authored-by: Cursor <cursoragent at cursor.com>
---
llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 14 +-
.../AMDGPUGlobalISelDivergenceLowering.cpp | 2 +-
.../Target/AMDGPU/AMDGPUGlobalISelUtils.cpp | 10 +-
.../lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h | 14 +-
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 15 +-
.../lib/Target/AMDGPU/AMDGPURegBankSelect.cpp | 7 +-
.../Target/AMDGPU/AMDGPUSearchableTables.td | 5 +-
.../Target/AMDGPU/SIAnnotateControlFlow.cpp | 23 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 +-
.../AMDGPU/control-flow-intrinsics.ll | 80 +-
.../AMDGPU/GlobalISel/atomicrmw_fmax.ll | 366 +-
.../AMDGPU/GlobalISel/atomicrmw_fmin.ll | 366 +-
...-divergent-i1-phis-no-lane-mask-merging.ll | 69 +-
...vergence-divergent-i1-used-outside-loop.ll | 227 +-
.../GlobalISel/divergence-structurizer.ll | 152 +-
.../divergence-temporal-divergent-i1.ll | 189 +-
.../divergence-temporal-divergent-reg.ll | 40 +-
.../GlobalISel/divergent-control-flow.ll | 30 +-
.../GlobalISel/global-atomic-fadd.f64.ll | 54 +-
.../GlobalISel/irtranslator-atomicrmw.ll | 40 +-
.../GlobalISel/irtranslator-function-args.ll | 4 +-
.../GlobalISel/llvm.amdgcn.end.cf.i32.ll | 10 +-
.../GlobalISel/llvm.amdgcn.end.cf.i64.ll | 12 +-
.../GlobalISel/llvm.amdgcn.if.break.i32.ll | 21 +-
.../GlobalISel/llvm.amdgcn.if.break.i64.ll | 29 +-
.../GlobalISel/llvm.amdgcn.wqm.demote.ll | 110 +-
.../regbankselect-mui-regbanklegalize.mir | 369 +-
.../AMDGPU/GlobalISel/regbankselect-mui.mir | 545 +--
.../test/CodeGen/AMDGPU/asyncmark-pregfx12.ll | 66 +-
.../CodeGen/AMDGPU/atomicrmw_usub_cond.ll | 130 +-
.../test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll | 1116 +++++--
.../buffer-fat-pointer-atomicrmw-fadd.ll | 235 +-
.../buffer-fat-pointer-atomicrmw-fmax.ll | 215 +-
.../buffer-fat-pointer-atomicrmw-fmin.ll | 215 +-
.../codegen-prepare-addrspacecast-non-null.ll | 95 +-
llvm/test/CodeGen/AMDGPU/div_i128.ll | 68 +-
llvm/test/CodeGen/AMDGPU/div_v2i128.ll | 376 +--
.../CodeGen/AMDGPU/diverge-switch-default.ll | 6 +-
.../divergent-branch-uniform-condition.ll | 8 +-
.../AMDGPU/divergent-loop-i1-lane-mask.ll | 34 +-
.../test/CodeGen/AMDGPU/flat-saddr-atomics.ll | 1340 +++++---
.../irtranslator-whole-wave-functions.ll | 4 +-
llvm/test/CodeGen/AMDGPU/literal64.ll | 81 +-
.../AMDGPU/llvm.amdgcn.init.whole.wave-w32.ll | 35 +-
.../llvm.amdgcn.raw.atomic.buffer.load.ll | 2781 ++++++++++++----
.../llvm.amdgcn.raw.ptr.atomic.buffer.load.ll | 2781 ++++++++++++----
.../llvm.amdgcn.struct.atomic.buffer.load.ll | 2960 +++++++++++++----
...vm.amdgcn.struct.ptr.atomic.buffer.load.ll | 2960 +++++++++++++----
.../AMDGPU/loop-on-function-argument.ll | 8 +-
llvm/test/CodeGen/AMDGPU/loop_break.ll | 48 +-
...ne-sink-temporal-divergence-swdev407790.ll | 2 +-
.../memory-legalizer-atomic-insert-end.mir | 12 +-
...er-multiple-mem-operands-nontemporal-1.mir | 10 +-
.../AMDGPU/memset-param-combinations.ll | 459 ++-
llvm/test/CodeGen/AMDGPU/memset-pattern.ll | 1014 ++++--
.../AMDGPU/multi-divergent-exit-region.ll | 90 +-
llvm/test/CodeGen/AMDGPU/multilevel-break.ll | 32 +-
.../CodeGen/AMDGPU/nested-loop-conditions.ll | 58 +-
llvm/test/CodeGen/AMDGPU/rem_i128.ll | 34 +-
llvm/test/CodeGen/AMDGPU/sdiv64.ll | 10 +-
.../AMDGPU/si-annotate-cf-unreachable.ll | 2 +-
.../CodeGen/AMDGPU/si-annotate-dbg-info.ll | 30 +-
.../AMDGPU/si-annotate-loop-i1-lane-mask.ll | 8 +-
.../si-annotatecfg-multiple-backedges.ll | 18 +-
.../si-unify-exit-return-unreachable.ll | 46 +-
llvm/test/CodeGen/AMDGPU/srem64.ll | 10 +-
llvm/test/CodeGen/AMDGPU/udiv64.ll | 10 +-
.../AMDGPU/unstructured-cfg-def-use-issue.ll | 42 +-
llvm/test/CodeGen/AMDGPU/urem64.ll | 8 +-
llvm/test/CodeGen/AMDGPU/valu-i1.ll | 22 +-
llvm/test/CodeGen/AMDGPU/wave32.ll | 10 +-
llvm/test/CodeGen/MIR/AMDGPU/syncscopes.mir | 10 +-
72 files changed, 14600 insertions(+), 5706 deletions(-)
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 21882e247c027..12c96d354760b 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -3903,24 +3903,24 @@ def int_amdgcn_cooperative_atomic_store_8x16B : AMDGPUCooperativeAtomicStore<llv
// convergence control tokens. The corresponding MIR pseudos are marked as
// having side effects, which is sufficient to prevent optimizations without
// having to mark them as convergent.
-def int_amdgcn_if : Intrinsic<[llvm_i1_ty, llvm_anyint_ty],
+def int_amdgcn_if : Intrinsic<[llvm_i1_ty, llvm_i1_ty],
[llvm_i1_ty], [IntrWillReturn, IntrNoCallback, IntrNoFree]
>;
-def int_amdgcn_else : Intrinsic<[llvm_i1_ty, llvm_anyint_ty],
- [llvm_anyint_ty], [IntrWillReturn, IntrNoCallback, IntrNoFree]
+def int_amdgcn_else : Intrinsic<[llvm_i1_ty, llvm_i1_ty],
+ [llvm_i1_ty], [IntrWillReturn, IntrNoCallback, IntrNoFree]
>;
-def int_amdgcn_if_break : Intrinsic<[llvm_anyint_ty],
- [llvm_i1_ty, LLVMMatchType<0>],
+def int_amdgcn_if_break : Intrinsic<[llvm_i1_ty],
+ [llvm_i1_ty, llvm_i1_ty],
[IntrNoMem, IntrWillReturn, IntrNoCallback, IntrNoFree]
>;
def int_amdgcn_loop : Intrinsic<[llvm_i1_ty],
- [llvm_anyint_ty], [IntrWillReturn, IntrNoCallback, IntrNoFree]
+ [llvm_i1_ty], [IntrWillReturn, IntrNoCallback, IntrNoFree]
>;
-def int_amdgcn_end_cf : Intrinsic<[], [llvm_anyint_ty],
+def int_amdgcn_end_cf : Intrinsic<[], [llvm_i1_ty],
[IntrWillReturn, IntrNoCallback, IntrNoFree]>;
// Represent unreachable in a divergent region.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp
index cbb4269e17260..2ae63025d2fd7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp
@@ -208,7 +208,7 @@ bool DivergenceLoweringHelper::lowerTemporalDivergence() {
for (auto [Reg, UseInst, _] : MUI->getTemporalDivergenceList()) {
if (MRI->getType(Reg) == LLT::scalar(1) || MUI->isDivergentAtDef(Reg) ||
- ILMA.isS32S64LaneMask(Reg))
+ ILMA.isLaneMask(Reg))
continue;
Register CachedTDCopy = TDCache.lookup(Reg);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
index f36935d8c0e8f..0b7b2fe8cb13b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
@@ -83,8 +83,8 @@ IntrinsicLaneMaskAnalyzer::IntrinsicLaneMaskAnalyzer(MachineFunction &MF)
initLaneMaskIntrinsics(MF);
}
-bool IntrinsicLaneMaskAnalyzer::isS32S64LaneMask(Register Reg) const {
- return S32S64LaneMask.contains(Reg);
+bool IntrinsicLaneMaskAnalyzer::isLaneMask(Register Reg) const {
+ return LaneMask.contains(Reg);
}
void IntrinsicLaneMaskAnalyzer::initLaneMaskIntrinsics(MachineFunction &MF) {
@@ -92,13 +92,13 @@ void IntrinsicLaneMaskAnalyzer::initLaneMaskIntrinsics(MachineFunction &MF) {
for (auto &MI : MBB) {
GIntrinsic *GI = dyn_cast<GIntrinsic>(&MI);
if (GI && GI->is(Intrinsic::amdgcn_if_break)) {
- S32S64LaneMask.insert(MI.getOperand(3).getReg());
- S32S64LaneMask.insert(MI.getOperand(0).getReg());
+ LaneMask.insert(MI.getOperand(3).getReg());
+ LaneMask.insert(MI.getOperand(0).getReg());
}
if (MI.getOpcode() == AMDGPU::SI_IF ||
MI.getOpcode() == AMDGPU::SI_ELSE) {
- S32S64LaneMask.insert(MI.getOperand(0).getReg());
+ LaneMask.insert(MI.getOperand(0).getReg());
}
}
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h
index 5e1000ee0ab26..40e7be7f7bbd7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h
@@ -31,19 +31,17 @@ getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg,
GISelValueTracking *ValueTracking = nullptr,
bool CheckNUW = false);
-// Currently finds S32/S64 lane masks that can be declared as divergent by
-// uniformity analysis (all are phis at the moment).
-// These are defined as i32/i64 in some IR intrinsics (not as i1).
-// Tablegen forces(via telling that lane mask IR intrinsics are uniform) most of
-// S32/S64 lane masks to be uniform, as this results in them ending up with sgpr
-// reg class after instruction-select, don't search for all of them.
+// Finds lane masks produced/consumed by the control flow intrinsics. These are
+// i1 values that live in wave-width lane mask registers (SReg_1). They are used
+// to assign such values to the Vcc (lane mask) register bank so that they
+// select to a wave mask register class.
class IntrinsicLaneMaskAnalyzer {
- SmallDenseSet<Register, 8> S32S64LaneMask;
+ SmallDenseSet<Register, 8> LaneMask;
MachineRegisterInfo &MRI;
public:
IntrinsicLaneMaskAnalyzer(MachineFunction &MF);
- bool isS32S64LaneMask(Register Reg) const;
+ bool isLaneMask(Register Reg) const;
private:
void initLaneMaskIntrinsics(MachineFunction &MF);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 7f7225ad6d311..acc71206e8cd3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -1875,14 +1875,13 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Any({{UniS1, _, S64}, {{UniInVcc}, {IntrId, Vgpr64, Vgpr32}}})
.Any({{DivS1, _, S64}, {{Vcc}, {IntrId, Vgpr64, Vgpr32}}});
- // This is "intrinsic lane mask" it was set to i32/i64 in llvm-ir.
- addRulesForIOpcs({amdgcn_end_cf})
- .Any({{_, UniS32}, {{}, {IntrId, Sgpr32}}})
- .Any({{_, UniS64}, {{}, {IntrId, Sgpr64}}});
-
- addRulesForIOpcs({amdgcn_if_break}, Standard)
- .Uni(S64, {{Sgpr64}, {IntrId, Vcc, Sgpr64}})
- .Uni(S32, {{Sgpr32}, {IntrId, Vcc, Sgpr32}});
+ // The "intrinsic lane mask" is a divergent i1 value living in a wave-width
+ // lane mask register (the Vcc bank / SReg_1). Modeling it as a divergent i1
+ // lets the existing divergent-i1-phi lowering thread the "phi.broken" mask
+ // through loops as a lane mask.
+ addRulesForIOpcs({amdgcn_end_cf}).Any({{_, S1}, {{}, {IntrId, Vcc}}});
+
+ addRulesForIOpcs({amdgcn_if_break}).Any({{S1}, {{Vcc}, {IntrId, Vcc, Vcc}}});
addRulesForIOpcs({amdgcn_exp})
.Any({{_, _, _, S32, S32, S32, S32},
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankSelect.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankSelect.cpp
index 247522a0dd43b..dc6bfa9e83bda 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankSelect.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankSelect.cpp
@@ -110,8 +110,13 @@ class RegBankSelectHelper {
}
const RegisterBank *getRegBankToAssign(Register Reg) {
+ // An "intrinsic lane mask" is an s1 value that lives in a wave-width lane
+ // mask register. Assign it to the Vcc (lane mask) bank rather than a scalar
+ // sgpr boolean so that it selects to a wave mask register class.
+ if (ILMA.isLaneMask(Reg) && MRI.getType(Reg) == LLT::scalar(1))
+ return VccRB;
if (!isTemporalDivergenceCopy(Reg) &&
- (MUI.isUniformAtDef(Reg) || ILMA.isS32S64LaneMask(Reg)))
+ (MUI.isUniformAtDef(Reg) || ILMA.isLaneMask(Reg)))
return SgprRB;
if (MRI.getType(Reg) == LLT::scalar(1))
return VccRB;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
index 2f7f72bd58f0e..df8a56ef51ead 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
@@ -376,6 +376,10 @@ def : SourceOfDivergence<int_amdgcn_ds_read_tr16_b64>;
def : SourceOfDivergence<int_amdgcn_if>;
def : SourceOfDivergence<int_amdgcn_else>;
def : SourceOfDivergence<int_amdgcn_loop>;
+// The "intrinsic lane mask" result is an i1 value that lives in a wave-width
+// lane mask register (SReg_1), i.e. a divergent i1 from the analysis's point of
+// view. This lets the forwarding phis be treated as lane masks.
+def : SourceOfDivergence<int_amdgcn_if_break>;
def : SourceOfDivergence<int_amdgcn_inverse_ballot>;
foreach intr = AMDGPUImageDimAtomicIntrinsics in
@@ -401,7 +405,6 @@ def : AlwaysUniform<int_amdgcn_readlane>;
def : AlwaysUniform<int_amdgcn_icmp>;
def : AlwaysUniform<int_amdgcn_fcmp>;
def : AlwaysUniform<int_amdgcn_ballot>;
-def : AlwaysUniform<int_amdgcn_if_break>;
def : AlwaysUniform<int_amdgcn_cluster_workgroup_id_x>;
def : AlwaysUniform<int_amdgcn_cluster_workgroup_id_y>;
def : AlwaysUniform<int_amdgcn_cluster_workgroup_id_z>;
diff --git a/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp b/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
index a7f98e79ce61d..09578918def17 100644
--- a/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
+++ b/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
@@ -45,13 +45,10 @@ class SIAnnotateControlFlow {
Type *Boolean;
Type *Void;
- Type *IntMask;
- Type *ReturnStruct;
ConstantInt *BoolTrue;
ConstantInt *BoolFalse;
PoisonValue *BoolPoison;
- Constant *IntMaskZero;
Function *If = nullptr;
Function *Else = nullptr;
@@ -115,14 +112,10 @@ void SIAnnotateControlFlow::initialize(const GCNSubtarget &ST) {
Void = Type::getVoidTy(Context);
Boolean = Type::getInt1Ty(Context);
- IntMask = ST.isWave32() ? Type::getInt32Ty(Context)
- : Type::getInt64Ty(Context);
- ReturnStruct = StructType::get(Boolean, IntMask);
BoolTrue = ConstantInt::getTrue(Context);
BoolFalse = ConstantInt::getFalse(Context);
BoolPoison = PoisonValue::get(Boolean);
- IntMaskZero = ConstantInt::get(IntMask, 0);
}
/// Is the branch condition uniform or did the StructurizeCFG pass
@@ -188,7 +181,7 @@ bool SIAnnotateControlFlow::openIf(CondBrInst *Term) {
return false;
IRBuilder<> IRB(Term);
- Value *IfCall = IRB.CreateCall(getDecl(If, Intrinsic::amdgcn_if, IntMask),
+ Value *IfCall = IRB.CreateCall(getDecl(If, Intrinsic::amdgcn_if, {}),
{Term->getCondition()});
Value *Cond = IRB.CreateExtractValue(IfCall, {0});
Value *Mask = IRB.CreateExtractValue(IfCall, {1});
@@ -204,8 +197,8 @@ bool SIAnnotateControlFlow::insertElse(CondBrInst *Term) {
}
IRBuilder<> IRB(Term);
- Value *ElseCall = IRB.CreateCall(
- getDecl(Else, Intrinsic::amdgcn_else, {IntMask, IntMask}), {popSaved()});
+ Value *ElseCall =
+ IRB.CreateCall(getDecl(Else, Intrinsic::amdgcn_else, {}), {popSaved()});
Value *Cond = IRB.CreateExtractValue(ElseCall, {0});
Value *Mask = IRB.CreateExtractValue(ElseCall, {1});
Term->setCondition(Cond);
@@ -220,7 +213,7 @@ Value *SIAnnotateControlFlow::handleLoopCondition(Value *Cond, PHINode *Broken,
auto CreateBreak = [this, Cond, Broken](Instruction *I) -> CallInst * {
return IRBuilder<>(I).CreateCall(
- getDecl(IfBreak, Intrinsic::amdgcn_if_break, IntMask), {Cond, Broken});
+ getDecl(IfBreak, Intrinsic::amdgcn_if_break, {}), {Cond, Broken});
};
if (Instruction *Inst = dyn_cast<Instruction>(Cond)) {
@@ -267,7 +260,7 @@ bool SIAnnotateControlFlow::handleLoop(CondBrInst *Term) {
return false;
BasicBlock *Target = Term->getSuccessor(1);
- PHINode *Broken = PHINode::Create(IntMask, 0, "phi.broken");
+ PHINode *Broken = PHINode::Create(Boolean, 0, "phi.broken");
Broken->insertBefore(Target->begin());
Value *Cond = Term->getCondition();
@@ -275,7 +268,7 @@ bool SIAnnotateControlFlow::handleLoop(CondBrInst *Term) {
Value *Arg = handleLoopCondition(Cond, Broken, L, Term);
for (BasicBlock *Pred : predecessors(Target)) {
- Value *PHIValue = IntMaskZero;
+ Value *PHIValue = BoolFalse;
if (Pred == BB) // Remember the value of the previous iteration.
PHIValue = Arg;
// If the backedge from Pred to Target could be executed before the exit
@@ -287,7 +280,7 @@ bool SIAnnotateControlFlow::handleLoop(CondBrInst *Term) {
}
CallInst *LoopCall = IRBuilder<>(Term).CreateCall(
- getDecl(Loop, Intrinsic::amdgcn_loop, IntMask), {Arg});
+ getDecl(Loop, Intrinsic::amdgcn_loop, {}), {Arg});
Term->setCondition(LoopCall);
push(Term->getSuccessor(0), Arg);
@@ -332,7 +325,7 @@ bool SIAnnotateControlFlow::closeControlFlow(BasicBlock *BB) {
// condition, for now just avoid copying these DebugLocs so that stepping
// out of the then/else block in a debugger doesn't step to the condition.
IRB.SetCurrentDebugLocation(DebugLoc());
- IRB.CreateCall(getDecl(EndCf, Intrinsic::amdgcn_end_cf, IntMask), {Exec});
+ IRB.CreateCall(getDecl(EndCf, Intrinsic::amdgcn_end_cf, {}), {Exec});
}
return true;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 9cf33b4ccbb87..e11d321bd83b3 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -20648,8 +20648,8 @@ static bool hasCFUser(const Value *V, SmallPtrSet<const Value *, 16> &Visited,
// FIXME: We assume we never cast the mask results of a control flow
// intrinsic.
// Early exit if the type won't be consistent as a compile time hack.
- IntegerType *IT = dyn_cast<IntegerType>(V->getType());
- if (!IT || IT->getBitWidth() != WaveSize)
+ // The mask results of the control flow intrinsics are i1 lane masks.
+ if (!V->getType()->isIntegerTy(1))
return false;
if (!isa<Instruction>(V))
diff --git a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/control-flow-intrinsics.ll b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/control-flow-intrinsics.ll
index b92daa64040e4..a3c3b5f544ad6 100644
--- a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/control-flow-intrinsics.ll
+++ b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/control-flow-intrinsics.ll
@@ -1,102 +1,102 @@
; RUN: opt -mtriple amdgcn-mesa-mesa3d -passes='print<uniformity>' -disable-output %s 2>&1 | FileCheck %s
-; Tests control flow intrinsics that should be treated as uniform
+; Tests uniformity of the AMDGPU control flow intrinsics. Their lane-mask
+; operands and results are i1 values that are divergent (per-lane).
; CHECK: for function 'test_if_break':
; CHECK: DIVERGENT: %cond = icmp eq i32 %arg0, 0
-; CHECK-NOT: DIVERGENT
-; CHECK: ret void
-define amdgpu_ps void @test_if_break(i32 %arg0, i64 inreg %saved) {
+; CHECK: DIVERGENT: %break = call i1 @llvm.amdgcn.if.break(i1 %cond, i1 %saved)
+define amdgpu_ps void @test_if_break(i32 %arg0, i1 inreg %saved) {
entry:
%cond = icmp eq i32 %arg0, 0
- %break = call i64 @llvm.amdgcn.if.break.i64.i64(i1 %cond, i64 %saved)
- store volatile i64 %break, ptr addrspace(1) undef
+ %break = call i1 @llvm.amdgcn.if.break(i1 %cond, i1 %saved)
+ store volatile i1 %break, ptr addrspace(1) undef
ret void
}
; CHECK: for function 'test_if':
; CHECK: DIVERGENT: %cond = icmp eq i32 %arg0, 0
-; CHECK-NEXT: DIVERGENT: %if = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %cond)
-; CHECK-NEXT: DIVERGENT: %if.bool = extractvalue { i1, i64 } %if, 0
+; CHECK-NEXT: DIVERGENT: %if = call { i1, i1 } @llvm.amdgcn.if(i1 %cond)
+; CHECK-NEXT: DIVERGENT: %if.bool = extractvalue { i1, i1 } %if, 0
; CHECK-NOT: DIVERGENT
; CHECK: DIVERGENT: %if.bool.ext = zext i1 %if.bool to i32
define void @test_if(i32 %arg0) {
entry:
%cond = icmp eq i32 %arg0, 0
- %if = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %cond)
- %if.bool = extractvalue { i1, i64 } %if, 0
- %if.mask = extractvalue { i1, i64 } %if, 1
+ %if = call { i1, i1 } @llvm.amdgcn.if(i1 %cond)
+ %if.bool = extractvalue { i1, i1 } %if, 0
+ %if.mask = extractvalue { i1, i1 } %if, 1
%if.bool.ext = zext i1 %if.bool to i32
store volatile i32 %if.bool.ext, ptr addrspace(1) undef
- store volatile i64 %if.mask, ptr addrspace(1) undef
+ store volatile i1 %if.mask, ptr addrspace(1) undef
ret void
}
; The result should still be treated as divergent, even with a uniform source.
; CHECK: for function 'test_if_uniform':
; CHECK-NOT: DIVERGENT
-; CHECK: DIVERGENT: %if = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %cond)
-; CHECK-NEXT: DIVERGENT: %if.bool = extractvalue { i1, i64 } %if, 0
+; CHECK: DIVERGENT: %if = call { i1, i1 } @llvm.amdgcn.if(i1 %cond)
+; CHECK-NEXT: DIVERGENT: %if.bool = extractvalue { i1, i1 } %if, 0
; CHECK-NOT: DIVERGENT
; CHECK: DIVERGENT: %if.bool.ext = zext i1 %if.bool to i32
define amdgpu_ps void @test_if_uniform(i32 inreg %arg0) {
entry:
%cond = icmp eq i32 %arg0, 0
- %if = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %cond)
- %if.bool = extractvalue { i1, i64 } %if, 0
- %if.mask = extractvalue { i1, i64 } %if, 1
+ %if = call { i1, i1 } @llvm.amdgcn.if(i1 %cond)
+ %if.bool = extractvalue { i1, i1 } %if, 0
+ %if.mask = extractvalue { i1, i1 } %if, 1
%if.bool.ext = zext i1 %if.bool to i32
store volatile i32 %if.bool.ext, ptr addrspace(1) undef
- store volatile i64 %if.mask, ptr addrspace(1) undef
+ store volatile i1 %if.mask, ptr addrspace(1) undef
ret void
}
; CHECK: for function 'test_loop_uniform':
-; CHECK: DIVERGENT: %loop = call i1 @llvm.amdgcn.loop.i64(i64 %mask)
-define amdgpu_ps void @test_loop_uniform(i64 inreg %mask) {
+; CHECK: DIVERGENT: %loop = call i1 @llvm.amdgcn.loop(i1 %mask)
+define amdgpu_ps void @test_loop_uniform(i1 inreg %mask) {
entry:
- %loop = call i1 @llvm.amdgcn.loop.i64(i64 %mask)
+ %loop = call i1 @llvm.amdgcn.loop(i1 %mask)
%loop.ext = zext i1 %loop to i32
store volatile i32 %loop.ext, ptr addrspace(1) undef
ret void
}
; CHECK: for function 'test_else':
-; CHECK: DIVERGENT: %else = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %mask)
-; CHECK: DIVERGENT: %else.bool = extractvalue { i1, i64 } %else, 0
-; CHECK: {{^[ \t]+}}%else.mask = extractvalue { i1, i64 } %else, 1
-define amdgpu_ps void @test_else(i64 inreg %mask) {
+; CHECK: DIVERGENT: %else = call { i1, i1 } @llvm.amdgcn.else(i1 %mask)
+; CHECK: DIVERGENT: %else.bool = extractvalue { i1, i1 } %else, 0
+; CHECK: {{^[ \t]+}}%else.mask = extractvalue { i1, i1 } %else, 1
+define amdgpu_ps void @test_else(i1 inreg %mask) {
entry:
- %else = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %mask)
- %else.bool = extractvalue { i1, i64 } %else, 0
- %else.mask = extractvalue { i1, i64 } %else, 1
+ %else = call { i1, i1 } @llvm.amdgcn.else(i1 %mask)
+ %else.bool = extractvalue { i1, i1 } %else, 0
+ %else.mask = extractvalue { i1, i1 } %else, 1
%else.bool.ext = zext i1 %else.bool to i32
store volatile i32 %else.bool.ext, ptr addrspace(1) undef
- store volatile i64 %else.mask, ptr addrspace(1) undef
+ store volatile i1 %else.mask, ptr addrspace(1) undef
ret void
}
; This case is probably always broken
; CHECK: for function 'test_else_divergent_mask':
-; CHECK: DIVERGENT: %if = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %mask)
-; CHECK-NEXT: DIVERGENT: %if.bool = extractvalue { i1, i64 } %if, 0
+; CHECK: DIVERGENT: %if = call { i1, i1 } @llvm.amdgcn.else(i1 %mask)
+; CHECK-NEXT: DIVERGENT: %if.bool = extractvalue { i1, i1 } %if, 0
; CHECK-NOT: DIVERGENT
; CHECK: DIVERGENT: %if.bool.ext = zext i1 %if.bool to i32
-define void @test_else_divergent_mask(i64 %mask) {
+define void @test_else_divergent_mask(i1 %mask) {
entry:
- %if = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %mask)
- %if.bool = extractvalue { i1, i64 } %if, 0
- %if.mask = extractvalue { i1, i64 } %if, 1
+ %if = call { i1, i1 } @llvm.amdgcn.else(i1 %mask)
+ %if.bool = extractvalue { i1, i1 } %if, 0
+ %if.mask = extractvalue { i1, i1 } %if, 1
%if.bool.ext = zext i1 %if.bool to i32
store volatile i32 %if.bool.ext, ptr addrspace(1) undef
- store volatile i64 %if.mask, ptr addrspace(1) undef
+ store volatile i1 %if.mask, ptr addrspace(1) undef
ret void
}
-declare { i1, i64 } @llvm.amdgcn.if.i64(i1) #0
-declare { i1, i64 } @llvm.amdgcn.else.i64.i64(i64) #0
-declare i64 @llvm.amdgcn.if.break.i64.i64(i1, i64) #1
-declare i1 @llvm.amdgcn.loop.i64(i64) #1
+declare { i1, i1 } @llvm.amdgcn.if(i1) #0
+declare { i1, i1 } @llvm.amdgcn.else(i1) #0
+declare i1 @llvm.amdgcn.if.break(i1, i1) #1
+declare i1 @llvm.amdgcn.loop(i1) #1
attributes #0 = { convergent nounwind }
attributes #1 = { convergent nounwind readnone }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll
index a3d5f1ee66ad3..6861c83c79290 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll
@@ -326,6 +326,7 @@ define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
; GFX942-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX942-NEXT: ; implicit-def: $sgpr2_sgpr3
; GFX942-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -339,10 +340,13 @@ define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
+; GFX942-NEXT: s_and_b64 s[4:5], exec, s[0:1]
+; GFX942-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB4_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v0, v3
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
@@ -372,6 +376,7 @@ define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX90A-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX90A-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -383,10 +388,13 @@ define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB4_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -396,6 +404,7 @@ define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
@@ -407,10 +416,13 @@ define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB4_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
@@ -420,6 +432,7 @@ define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
@@ -431,10 +444,13 @@ define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB4_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -473,6 +489,7 @@ define void @global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX942-NEXT: ; implicit-def: $sgpr2_sgpr3
; GFX942-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -485,11 +502,14 @@ define void @global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
+; GFX942-NEXT: s_and_b64 s[4:5], exec, s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB5_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
@@ -518,6 +538,7 @@ define void @global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX90A-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX90A-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -528,11 +549,14 @@ define void @global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB5_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
@@ -541,6 +565,7 @@ define void @global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
@@ -551,11 +576,14 @@ define void @global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB5_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
@@ -564,6 +592,7 @@ define void @global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
@@ -574,11 +603,14 @@ define void @global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB5_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
@@ -606,6 +638,7 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: ; implicit-def: $sgpr1
; GFX12-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -620,11 +653,16 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_and_not1_b32 s1, s1, exec_lo
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s1, s1, s2
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB6_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
@@ -644,6 +682,7 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: ; implicit-def: $sgpr1
; GFX11-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -658,11 +697,14 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_and_b32 s2, exec_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s1, s1, s2
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB6_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
@@ -690,6 +732,7 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
@@ -702,10 +745,13 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB6_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v4
; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
@@ -716,6 +762,7 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
@@ -728,10 +775,13 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB6_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v4
; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -763,6 +813,7 @@ define void @global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: ; implicit-def: $sgpr1
; GFX12-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -777,11 +828,16 @@ define void @global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_b32 s2, exec_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s1, s1, s2
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB7_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
@@ -800,6 +856,7 @@ define void @global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: ; implicit-def: $sgpr1
; GFX11-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -814,11 +871,14 @@ define void @global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_and_b32 s2, exec_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s1, s1, s2
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB7_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
@@ -845,6 +905,7 @@ define void @global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
@@ -856,11 +917,14 @@ define void @global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v2
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB7_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
@@ -869,6 +933,7 @@ define void @global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
@@ -880,11 +945,14 @@ define void @global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v2
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB7_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
@@ -921,6 +989,7 @@ define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
; GFX942-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX942-NEXT: ; implicit-def: $sgpr2_sgpr3
; GFX942-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -934,10 +1003,13 @@ define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
+; GFX942-NEXT: s_and_b64 s[4:5], exec, s[0:1]
+; GFX942-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB8_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v0, v3
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
@@ -967,6 +1039,7 @@ define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX90A-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX90A-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -978,10 +1051,13 @@ define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB8_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -991,6 +1067,7 @@ define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1002,10 +1079,13 @@ define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB8_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
@@ -1015,6 +1095,7 @@ define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1026,10 +1107,13 @@ define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB8_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -1064,6 +1148,7 @@ define void @flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX942-NEXT: ; implicit-def: $sgpr2_sgpr3
; GFX942-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1076,11 +1161,14 @@ define void @flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
+; GFX942-NEXT: s_and_b64 s[4:5], exec, s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB9_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
@@ -1111,6 +1199,7 @@ define void @flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX90A-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX90A-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1121,11 +1210,14 @@ define void @flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB9_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
@@ -1134,6 +1226,7 @@ define void @flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1144,11 +1237,14 @@ define void @flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB9_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
@@ -1157,6 +1253,7 @@ define void @flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1167,11 +1264,14 @@ define void @flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB9_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
@@ -1196,6 +1296,7 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: ; implicit-def: $sgpr1
; GFX12-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1210,11 +1311,16 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_b32 s2, exec_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s1, s1, s2
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB10_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
@@ -1234,6 +1340,7 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: ; implicit-def: $sgpr1
; GFX11-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1248,11 +1355,14 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_and_b32 s2, exec_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s1, s1, s2
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
@@ -1280,6 +1390,7 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1292,10 +1403,13 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB10_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v4
; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
@@ -1309,6 +1423,7 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: flat_load_dword v5, v[5:6]
; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1321,10 +1436,13 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB10_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v4
; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -1351,6 +1469,7 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: ; implicit-def: $sgpr1
; GFX12-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1365,11 +1484,16 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_and_not1_b32 s1, s1, exec_lo
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s1, s1, s2
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB11_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
@@ -1388,6 +1512,7 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: ; implicit-def: $sgpr1
; GFX11-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1402,11 +1527,14 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_and_b32 s2, exec_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s1, s1, s2
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB11_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
@@ -1434,6 +1562,7 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1445,11 +1574,14 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v2
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB11_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
@@ -1461,6 +1593,7 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: flat_load_dword v5, v[5:6]
; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1472,11 +1605,14 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v2
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB11_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
@@ -1512,10 +1648,11 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX942-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX942-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_readfirstlane_b32 s6, v1
+; GFX942-NEXT: v_readfirstlane_b32 s8, v1
; GFX942-NEXT: s_nop 1
-; GFX942-NEXT: v_mov_b32_e32 v0, s6
+; GFX942-NEXT: v_mov_b32_e32 v0, s8
; GFX942-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: v_mov_b32_e32 v5, v0
@@ -1529,10 +1666,13 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX942-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX942-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_cbranch_execnz .LBB12_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
@@ -1564,9 +1704,10 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX90A-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX90A-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_readfirstlane_b32 s6, v1
-; GFX90A-NEXT: v_mov_b32_e32 v0, s6
+; GFX90A-NEXT: v_readfirstlane_b32 s8, v1
+; GFX90A-NEXT: v_mov_b32_e32 v0, s8
; GFX90A-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
@@ -1578,10 +1719,13 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB12_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
@@ -1591,9 +1735,10 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX908-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_readfirstlane_b32 s6, v1
-; GFX908-NEXT: v_mov_b32_e32 v0, s6
+; GFX908-NEXT: v_readfirstlane_b32 s8, v1
+; GFX908-NEXT: v_mov_b32_e32 v0, s8
; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_mov_b32_e32 v5, v0
@@ -1606,10 +1751,13 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB12_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
@@ -1619,9 +1767,10 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX8-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v0
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_readfirstlane_b32 s6, v1
-; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: v_readfirstlane_b32 s8, v1
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
; GFX8-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_mov_b32_e32 v5, v0
@@ -1634,10 +1783,13 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB12_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
@@ -1674,10 +1826,11 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX942-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX942-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_readfirstlane_b32 s6, v1
+; GFX942-NEXT: v_readfirstlane_b32 s8, v1
; GFX942-NEXT: s_nop 1
-; GFX942-NEXT: v_mov_b32_e32 v1, s6
+; GFX942-NEXT: v_mov_b32_e32 v1, s8
; GFX942-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: v_max_f32_e32 v0, v1, v1
@@ -1690,11 +1843,14 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX942-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX942-NEXT: v_mov_b32_e32 v1, v4
+; GFX942-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_cbranch_execnz .LBB13_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
@@ -1726,9 +1882,10 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX90A-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX90A-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_readfirstlane_b32 s6, v1
-; GFX90A-NEXT: v_mov_b32_e32 v1, s6
+; GFX90A-NEXT: v_readfirstlane_b32 s8, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, s8
; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
@@ -1739,11 +1896,14 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v4
+; GFX90A-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
@@ -1753,9 +1913,10 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX908-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_readfirstlane_b32 s6, v1
-; GFX908-NEXT: v_mov_b32_e32 v1, s6
+; GFX908-NEXT: v_readfirstlane_b32 s8, v1
+; GFX908-NEXT: v_mov_b32_e32 v1, s8
; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_max_f32_e32 v0, v1, v1
@@ -1767,11 +1928,14 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v1, v4
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB13_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
@@ -1781,9 +1945,10 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX8-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v0
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_readfirstlane_b32 s6, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
+; GFX8-NEXT: v_readfirstlane_b32 s8, v1
+; GFX8-NEXT: v_mov_b32_e32 v1, s8
; GFX8-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v1
@@ -1795,11 +1960,14 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v1, v4
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB13_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
@@ -1832,6 +2000,7 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
; GFX12-NEXT: s_mov_b32 s4, 0
+; GFX12-NEXT: ; implicit-def: $sgpr5
; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -1848,11 +2017,16 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_b32 s6, exec_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s5, s5, s6
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_execnz .LBB14_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
@@ -1878,6 +2052,8 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: ; implicit-def: $sgpr5
+; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -1895,11 +2071,14 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-NEXT: s_and_b32 s6, exec_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s5, s5, s6
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB14_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
@@ -1929,11 +2108,12 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX908-NEXT: buffer_load_dwordx2 v[2:3], v8, s[16:19], 0 offen
; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_readfirstlane_b32 s6, v2
-; GFX908-NEXT: v_readfirstlane_b32 s7, v3
-; GFX908-NEXT: v_mov_b32_e32 v4, s6
-; GFX908-NEXT: v_mov_b32_e32 v5, s7
+; GFX908-NEXT: v_readfirstlane_b32 s8, v2
+; GFX908-NEXT: v_readfirstlane_b32 s9, v3
+; GFX908-NEXT: v_mov_b32_e32 v4, s8
+; GFX908-NEXT: v_mov_b32_e32 v5, s9
; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
@@ -1948,11 +2128,14 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v0
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB14_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
@@ -1962,11 +2145,12 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX8-NEXT: buffer_load_dwordx2 v[2:3], v8, s[16:19], 0 offen
; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_readfirstlane_b32 s6, v2
-; GFX8-NEXT: v_readfirstlane_b32 s7, v3
-; GFX8-NEXT: v_mov_b32_e32 v4, s6
-; GFX8-NEXT: v_mov_b32_e32 v5, s7
+; GFX8-NEXT: v_readfirstlane_b32 s8, v2
+; GFX8-NEXT: v_readfirstlane_b32 s9, v3
+; GFX8-NEXT: v_mov_b32_e32 v4, s8
+; GFX8-NEXT: v_mov_b32_e32 v5, s9
; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
@@ -1981,11 +2165,14 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB14_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
@@ -2018,6 +2205,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT: s_mov_b32 s4, 0
+; GFX12-NEXT: ; implicit-def: $sgpr5
; GFX12-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -2033,11 +2221,16 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
; GFX12-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_b32 s6, exec_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s5, s5, s6
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_execnz .LBB15_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
@@ -2063,6 +2256,8 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: ; implicit-def: $sgpr5
+; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -2079,11 +2274,14 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
; GFX11-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-NEXT: s_and_b32 s6, exec_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s5, s5, s6
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
@@ -2113,11 +2311,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX908-NEXT: buffer_load_dwordx2 v[2:3], v6, s[16:19], 0 offen
; GFX908-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_readfirstlane_b32 s6, v2
-; GFX908-NEXT: v_readfirstlane_b32 s7, v3
-; GFX908-NEXT: v_mov_b32_e32 v2, s6
-; GFX908-NEXT: v_mov_b32_e32 v3, s7
+; GFX908-NEXT: v_readfirstlane_b32 s8, v2
+; GFX908-NEXT: v_readfirstlane_b32 s9, v3
+; GFX908-NEXT: v_mov_b32_e32 v2, s8
+; GFX908-NEXT: v_mov_b32_e32 v3, s9
; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
@@ -2132,11 +2331,14 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[2:3]
; GFX908-NEXT: v_mov_b32_e32 v2, v7
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v8
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB15_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
@@ -2146,11 +2348,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX8-NEXT: buffer_load_dwordx2 v[2:3], v6, s[16:19], 0 offen
; GFX8-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_readfirstlane_b32 s6, v2
-; GFX8-NEXT: v_readfirstlane_b32 s7, v3
-; GFX8-NEXT: v_mov_b32_e32 v2, s6
-; GFX8-NEXT: v_mov_b32_e32 v3, s7
+; GFX8-NEXT: v_readfirstlane_b32 s8, v2
+; GFX8-NEXT: v_readfirstlane_b32 s9, v3
+; GFX8-NEXT: v_mov_b32_e32 v2, s8
+; GFX8-NEXT: v_mov_b32_e32 v3, s9
; GFX8-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
@@ -2165,11 +2368,14 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v2, v7
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v8
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB15_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll
index a534dd306cb47..704f733f005bc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll
@@ -326,6 +326,7 @@ define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
; GFX942-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX942-NEXT: ; implicit-def: $sgpr2_sgpr3
; GFX942-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -339,10 +340,13 @@ define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
+; GFX942-NEXT: s_and_b64 s[4:5], exec, s[0:1]
+; GFX942-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB4_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v0, v3
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
@@ -372,6 +376,7 @@ define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX90A-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX90A-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -383,10 +388,13 @@ define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB4_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -396,6 +404,7 @@ define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
@@ -407,10 +416,13 @@ define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB4_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
@@ -420,6 +432,7 @@ define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
@@ -431,10 +444,13 @@ define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB4_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -473,6 +489,7 @@ define void @global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX942-NEXT: ; implicit-def: $sgpr2_sgpr3
; GFX942-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -485,11 +502,14 @@ define void @global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
+; GFX942-NEXT: s_and_b64 s[4:5], exec, s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB5_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
@@ -518,6 +538,7 @@ define void @global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX90A-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX90A-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -528,11 +549,14 @@ define void @global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB5_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
@@ -541,6 +565,7 @@ define void @global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
@@ -551,11 +576,14 @@ define void @global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB5_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
@@ -564,6 +592,7 @@ define void @global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
@@ -574,11 +603,14 @@ define void @global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB5_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
@@ -606,6 +638,7 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: ; implicit-def: $sgpr1
; GFX12-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -620,11 +653,16 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_and_not1_b32 s1, s1, exec_lo
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s1, s1, s2
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB6_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
@@ -644,6 +682,7 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: ; implicit-def: $sgpr1
; GFX11-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -658,11 +697,14 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_and_b32 s2, exec_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s1, s1, s2
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB6_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
@@ -690,6 +732,7 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
@@ -702,10 +745,13 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB6_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v4
; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
@@ -716,6 +762,7 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
@@ -728,10 +775,13 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB6_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v4
; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -763,6 +813,7 @@ define void @global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: ; implicit-def: $sgpr1
; GFX12-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -777,11 +828,16 @@ define void @global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_b32 s2, exec_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s1, s1, s2
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB7_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
@@ -800,6 +856,7 @@ define void @global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: ; implicit-def: $sgpr1
; GFX11-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -814,11 +871,14 @@ define void @global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_and_b32 s2, exec_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s1, s1, s2
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB7_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
@@ -845,6 +905,7 @@ define void @global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
@@ -856,11 +917,14 @@ define void @global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v2
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB7_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
@@ -869,6 +933,7 @@ define void @global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
@@ -880,11 +945,14 @@ define void @global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v2
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB7_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
@@ -921,6 +989,7 @@ define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
; GFX942-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX942-NEXT: ; implicit-def: $sgpr2_sgpr3
; GFX942-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -934,10 +1003,13 @@ define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
+; GFX942-NEXT: s_and_b64 s[4:5], exec, s[0:1]
+; GFX942-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB8_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v0, v3
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
@@ -967,6 +1039,7 @@ define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX90A-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX90A-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -978,10 +1051,13 @@ define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB8_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -991,6 +1067,7 @@ define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1002,10 +1079,13 @@ define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB8_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
@@ -1015,6 +1095,7 @@ define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1026,10 +1107,13 @@ define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB8_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -1064,6 +1148,7 @@ define void @flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX942-NEXT: ; implicit-def: $sgpr2_sgpr3
; GFX942-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1076,11 +1161,14 @@ define void @flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
+; GFX942-NEXT: s_and_b64 s[4:5], exec, s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB9_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
@@ -1111,6 +1199,7 @@ define void @flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX90A-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX90A-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1121,11 +1210,14 @@ define void @flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB9_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
@@ -1134,6 +1226,7 @@ define void @flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1144,11 +1237,14 @@ define void @flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB9_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
@@ -1157,6 +1253,7 @@ define void @flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1167,11 +1264,14 @@ define void @flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v2
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB9_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
@@ -1196,6 +1296,7 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: ; implicit-def: $sgpr1
; GFX12-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1210,11 +1311,16 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_b32 s2, exec_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s1, s1, s2
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB10_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
@@ -1234,6 +1340,7 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: ; implicit-def: $sgpr1
; GFX11-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1248,11 +1355,14 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_and_b32 s2, exec_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s1, s1, s2
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
@@ -1280,6 +1390,7 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1292,10 +1403,13 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB10_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v4
; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
@@ -1309,6 +1423,7 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: flat_load_dword v5, v[5:6]
; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1321,10 +1436,13 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB10_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v4
; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -1351,6 +1469,7 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: ; implicit-def: $sgpr1
; GFX12-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1365,11 +1484,16 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_and_not1_b32 s1, s1, exec_lo
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s1, s1, s2
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB11_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
@@ -1388,6 +1512,7 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: ; implicit-def: $sgpr1
; GFX11-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1402,11 +1527,14 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_and_b32 s2, exec_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s1, s1, s2
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB11_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
@@ -1434,6 +1562,7 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1445,11 +1574,14 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v2
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB11_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
@@ -1461,6 +1593,7 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: flat_load_dword v5, v[5:6]
; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1472,11 +1605,14 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v2
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB11_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
@@ -1512,10 +1648,11 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX942-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX942-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_readfirstlane_b32 s6, v1
+; GFX942-NEXT: v_readfirstlane_b32 s8, v1
; GFX942-NEXT: s_nop 1
-; GFX942-NEXT: v_mov_b32_e32 v0, s6
+; GFX942-NEXT: v_mov_b32_e32 v0, s8
; GFX942-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: v_mov_b32_e32 v5, v0
@@ -1529,10 +1666,13 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX942-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX942-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_cbranch_execnz .LBB12_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
@@ -1564,9 +1704,10 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX90A-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX90A-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_readfirstlane_b32 s6, v1
-; GFX90A-NEXT: v_mov_b32_e32 v0, s6
+; GFX90A-NEXT: v_readfirstlane_b32 s8, v1
+; GFX90A-NEXT: v_mov_b32_e32 v0, s8
; GFX90A-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
@@ -1578,10 +1719,13 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB12_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
@@ -1591,9 +1735,10 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX908-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_readfirstlane_b32 s6, v1
-; GFX908-NEXT: v_mov_b32_e32 v0, s6
+; GFX908-NEXT: v_readfirstlane_b32 s8, v1
+; GFX908-NEXT: v_mov_b32_e32 v0, s8
; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_mov_b32_e32 v5, v0
@@ -1606,10 +1751,13 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB12_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
@@ -1619,9 +1767,10 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX8-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v0
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_readfirstlane_b32 s6, v1
-; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: v_readfirstlane_b32 s8, v1
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
; GFX8-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_mov_b32_e32 v5, v0
@@ -1634,10 +1783,13 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB12_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
@@ -1674,10 +1826,11 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX942-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX942-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_readfirstlane_b32 s6, v1
+; GFX942-NEXT: v_readfirstlane_b32 s8, v1
; GFX942-NEXT: s_nop 1
-; GFX942-NEXT: v_mov_b32_e32 v1, s6
+; GFX942-NEXT: v_mov_b32_e32 v1, s8
; GFX942-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: v_max_f32_e32 v0, v1, v1
@@ -1690,11 +1843,14 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX942-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX942-NEXT: v_mov_b32_e32 v1, v4
+; GFX942-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_cbranch_execnz .LBB13_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
@@ -1726,9 +1882,10 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX90A-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX90A-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_readfirstlane_b32 s6, v1
-; GFX90A-NEXT: v_mov_b32_e32 v1, s6
+; GFX90A-NEXT: v_readfirstlane_b32 s8, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, s8
; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
@@ -1739,11 +1896,14 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v4
+; GFX90A-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
@@ -1753,9 +1913,10 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX908-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_max_f32_e32 v3, v0, v0
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_readfirstlane_b32 s6, v1
-; GFX908-NEXT: v_mov_b32_e32 v1, s6
+; GFX908-NEXT: v_readfirstlane_b32 s8, v1
+; GFX908-NEXT: v_mov_b32_e32 v1, s8
; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_max_f32_e32 v0, v1, v1
@@ -1767,11 +1928,14 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v1, v4
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB13_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
@@ -1781,9 +1945,10 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX8-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v0
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_readfirstlane_b32 s6, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
+; GFX8-NEXT: v_readfirstlane_b32 s8, v1
+; GFX8-NEXT: v_mov_b32_e32 v1, s8
; GFX8-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v1
@@ -1795,11 +1960,14 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v1, v4
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB13_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
@@ -1832,6 +2000,7 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
; GFX12-NEXT: s_mov_b32 s4, 0
+; GFX12-NEXT: ; implicit-def: $sgpr5
; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -1848,11 +2017,16 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_b32 s6, exec_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s5, s5, s6
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_execnz .LBB14_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
@@ -1878,6 +2052,8 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: ; implicit-def: $sgpr5
+; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -1895,11 +2071,14 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-NEXT: s_and_b32 s6, exec_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s5, s5, s6
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB14_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
@@ -1929,11 +2108,12 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX908-NEXT: buffer_load_dwordx2 v[2:3], v8, s[16:19], 0 offen
; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_readfirstlane_b32 s6, v2
-; GFX908-NEXT: v_readfirstlane_b32 s7, v3
-; GFX908-NEXT: v_mov_b32_e32 v4, s6
-; GFX908-NEXT: v_mov_b32_e32 v5, s7
+; GFX908-NEXT: v_readfirstlane_b32 s8, v2
+; GFX908-NEXT: v_readfirstlane_b32 s9, v3
+; GFX908-NEXT: v_mov_b32_e32 v4, s8
+; GFX908-NEXT: v_mov_b32_e32 v5, s9
; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
@@ -1948,11 +2128,14 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v0
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB14_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
@@ -1962,11 +2145,12 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX8-NEXT: buffer_load_dwordx2 v[2:3], v8, s[16:19], 0 offen
; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_readfirstlane_b32 s6, v2
-; GFX8-NEXT: v_readfirstlane_b32 s7, v3
-; GFX8-NEXT: v_mov_b32_e32 v4, s6
-; GFX8-NEXT: v_mov_b32_e32 v5, s7
+; GFX8-NEXT: v_readfirstlane_b32 s8, v2
+; GFX8-NEXT: v_readfirstlane_b32 s9, v3
+; GFX8-NEXT: v_mov_b32_e32 v4, s8
+; GFX8-NEXT: v_mov_b32_e32 v5, s9
; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
@@ -1981,11 +2165,14 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB14_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
@@ -2018,6 +2205,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT: s_mov_b32 s4, 0
+; GFX12-NEXT: ; implicit-def: $sgpr5
; GFX12-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -2033,11 +2221,16 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
; GFX12-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_b32 s6, exec_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s5, s5, s6
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_execnz .LBB15_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
@@ -2063,6 +2256,8 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: ; implicit-def: $sgpr5
+; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -2079,11 +2274,14 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
; GFX11-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-NEXT: s_and_b32 s6, exec_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s5, s5, s6
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
@@ -2113,11 +2311,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX908-NEXT: buffer_load_dwordx2 v[2:3], v6, s[16:19], 0 offen
; GFX908-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_readfirstlane_b32 s6, v2
-; GFX908-NEXT: v_readfirstlane_b32 s7, v3
-; GFX908-NEXT: v_mov_b32_e32 v2, s6
-; GFX908-NEXT: v_mov_b32_e32 v3, s7
+; GFX908-NEXT: v_readfirstlane_b32 s8, v2
+; GFX908-NEXT: v_readfirstlane_b32 s9, v3
+; GFX908-NEXT: v_mov_b32_e32 v2, s8
+; GFX908-NEXT: v_mov_b32_e32 v3, s9
; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
@@ -2132,11 +2331,14 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[2:3]
; GFX908-NEXT: v_mov_b32_e32 v2, v7
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX908-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v3, v8
+; GFX908-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB15_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
@@ -2146,11 +2348,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX8-NEXT: buffer_load_dwordx2 v[2:3], v6, s[16:19], 0 offen
; GFX8-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_readfirstlane_b32 s6, v2
-; GFX8-NEXT: v_readfirstlane_b32 s7, v3
-; GFX8-NEXT: v_mov_b32_e32 v2, s6
-; GFX8-NEXT: v_mov_b32_e32 v3, s7
+; GFX8-NEXT: v_readfirstlane_b32 s8, v2
+; GFX8-NEXT: v_readfirstlane_b32 s9, v3
+; GFX8-NEXT: v_mov_b32_e32 v2, s8
+; GFX8-NEXT: v_mov_b32_e32 v3, s9
; GFX8-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
@@ -2165,11 +2368,14 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v2, v7
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v8
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB15_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll
index 394c7c8914e6a..7c482abe82b06 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll
@@ -105,23 +105,27 @@ define void @divergent_i1_phi_used_inside_loop(float %val, ptr %addr) {
; GFX10-NEXT: s_mov_b32 s5, 1
; GFX10-NEXT: s_mov_b32 s6, 0
; GFX10-NEXT: ; implicit-def: $sgpr7
+; GFX10-NEXT: ; implicit-def: $sgpr8
; GFX10-NEXT: .LBB2_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_cvt_f32_u32_e32 v3, s6
-; GFX10-NEXT: s_and_b32 s8, s5, 1
-; GFX10-NEXT: s_cmp_lg_u32 s8, 0
-; GFX10-NEXT: s_cselect_b32 s8, exec_lo, 0
+; GFX10-NEXT: v_cvt_f32_u32_e32 v3, s4
+; GFX10-NEXT: s_and_b32 s9, s5, 1
+; GFX10-NEXT: s_cmp_lg_u32 s9, 0
+; GFX10-NEXT: s_cselect_b32 s9, exec_lo, 0
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v0
; GFX10-NEXT: s_xor_b32 s5, s5, 1
-; GFX10-NEXT: s_add_i32 s6, s6, 1
-; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_add_i32 s4, s4, 1
+; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX10-NEXT: s_andn2_b32 s8, s8, exec_lo
; GFX10-NEXT: s_andn2_b32 s7, s7, exec_lo
-; GFX10-NEXT: s_and_b32 s8, exec_lo, s8
-; GFX10-NEXT: s_or_b32 s7, s7, s8
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_and_b32 s9, exec_lo, s9
+; GFX10-NEXT: s_and_b32 s10, exec_lo, s6
+; GFX10-NEXT: s_or_b32 s7, s7, s9
+; GFX10-NEXT: s_or_b32 s8, s8, s10
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX10-NEXT: s_cbranch_execnz .LBB2_1
; GFX10-NEXT: ; %bb.2: ; %exit
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_cndmask_b32_e64 v0, 1.0, 0, s7
; GFX10-NEXT: flat_store_dword v[1:2], v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -153,41 +157,46 @@ define void @divergent_i1_phi_used_inside_loop_bigger_loop_body(float %val, floa
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_mov_b32 s6, 0
; GFX10-NEXT: ; implicit-def: $sgpr7
+; GFX10-NEXT: ; implicit-def: $sgpr8
; GFX10-NEXT: s_branch .LBB3_2
; GFX10-NEXT: .LBB3_1: ; %loop_body
; GFX10-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX10-NEXT: v_cvt_f32_u32_e32 v8, s6
-; GFX10-NEXT: s_xor_b32 s4, s4, exec_lo
-; GFX10-NEXT: s_add_i32 s6, s6, 1
+; GFX10-NEXT: v_cvt_f32_u32_e32 v8, s5
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_add_i32 s5, s5, 1
+; GFX10-NEXT: s_xor_b32 s4, s4, s9
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v0
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX10-NEXT: s_andn2_b32 s8, s8, exec_lo
+; GFX10-NEXT: s_and_b32 s9, exec_lo, s6
+; GFX10-NEXT: s_or_b32 s8, s8, s9
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX10-NEXT: s_cbranch_execz .LBB3_6
; GFX10-NEXT: .LBB3_2: ; %loop_start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_cmpk_le_i32 s6, 0x3e8
-; GFX10-NEXT: s_cselect_b32 s9, 1, 0
+; GFX10-NEXT: s_cmpk_le_i32 s5, 0x3e8
+; GFX10-NEXT: s_cselect_b32 s10, 1, 0
; GFX10-NEXT: s_andn2_b32 s7, s7, exec_lo
-; GFX10-NEXT: s_and_b32 s8, exec_lo, s4
-; GFX10-NEXT: s_or_b32 s7, s7, s8
-; GFX10-NEXT: s_mov_b32 s8, 1
-; GFX10-NEXT: s_cmp_lg_u32 s9, 0
+; GFX10-NEXT: s_and_b32 s9, exec_lo, s4
+; GFX10-NEXT: s_or_b32 s7, s7, s9
+; GFX10-NEXT: s_mov_b32 s9, 1
+; GFX10-NEXT: s_cmp_lg_u32 s10, 0
; GFX10-NEXT: s_cbranch_scc0 .LBB3_4
; GFX10-NEXT: ; %bb.3: ; %else
; GFX10-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s9, 0
; GFX10-NEXT: flat_store_dword v[6:7], v1
; GFX10-NEXT: .LBB3_4: ; %Flow
; GFX10-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX10-NEXT: s_xor_b32 s8, s8, 1
-; GFX10-NEXT: s_cmp_lg_u32 s8, 0
+; GFX10-NEXT: s_xor_b32 s9, s9, 1
+; GFX10-NEXT: s_cmp_lg_u32 s9, 0
; GFX10-NEXT: s_cbranch_scc1 .LBB3_1
; GFX10-NEXT: ; %bb.5: ; %if
; GFX10-NEXT: ; in Loop: Header=BB3_2 Depth=1
; GFX10-NEXT: flat_store_dword v[4:5], v1
; GFX10-NEXT: s_branch .LBB3_1
; GFX10-NEXT: .LBB3_6: ; %exit
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_cndmask_b32_e64 v0, 1.0, 0, s7
; GFX10-NEXT: flat_store_dword v[2:3], v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -249,6 +258,7 @@ define amdgpu_cs void @single_lane_execution_attribute(i32 inreg %.userdata0, <3
; GFX10-NEXT: ; %bb.1: ; %.preheader.preheader
; GFX10-NEXT: s_mov_b32 s12, 0
; GFX10-NEXT: s_mov_b32 s13, 0
+; GFX10-NEXT: ; implicit-def: $sgpr14
; GFX10-NEXT: .LBB4_2: ; %.preheader
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v3, s3
@@ -257,14 +267,17 @@ define amdgpu_cs void @single_lane_execution_attribute(i32 inreg %.userdata0, <3
; GFX10-NEXT: buffer_load_dword v3, v3, s[4:7], 0 offen
; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_readfirstlane_b32 s14, v3
-; GFX10-NEXT: s_add_i32 s13, s14, s13
+; GFX10-NEXT: v_readfirstlane_b32 s15, v3
+; GFX10-NEXT: s_add_i32 s13, s15, s13
; GFX10-NEXT: s_or_b32 s12, s0, s12
; GFX10-NEXT: v_mov_b32_e32 v3, s13
+; GFX10-NEXT: s_andn2_b32 s0, s14, exec_lo
+; GFX10-NEXT: s_and_b32 s14, exec_lo, s12
+; GFX10-NEXT: s_or_b32 s14, s0, s14
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s12
; GFX10-NEXT: s_cbranch_execnz .LBB4_2
; GFX10-NEXT: ; %bb.3: ; %.preheader._crit_edge
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s12
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s14
; GFX10-NEXT: v_cmp_eq_u32_e64 s0, v3, v1
; GFX10-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX10-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll
index 6a43cba0f9ec5..fe8d1934f89db 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll
@@ -13,26 +13,30 @@ define void @divergent_i1_phi_used_outside_loop(float %val, float %pre.cond.val,
; GFX10-LABEL: divergent_i1_phi_used_outside_loop:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cmp_lt_f32_e64 s5, 1.0, v1
+; GFX10-NEXT: v_cmp_lt_f32_e64 s6, 1.0, v1
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: s_mov_b32 s6, 0
+; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: ; implicit-def: $sgpr7
+; GFX10-NEXT: ; implicit-def: $sgpr8
; GFX10-NEXT: .LBB0_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s6
-; GFX10-NEXT: s_mov_b32 s8, exec_lo
-; GFX10-NEXT: s_add_i32 s6, s6, 1
-; GFX10-NEXT: s_xor_b32 s8, s5, s8
+; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s4
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_add_i32 s4, s4, 1
+; GFX10-NEXT: s_xor_b32 s9, s6, s9
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v0
-; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_andn2_b32 s8, s8, exec_lo
; GFX10-NEXT: s_andn2_b32 s7, s7, exec_lo
+; GFX10-NEXT: s_and_b32 s10, exec_lo, s6
+; GFX10-NEXT: s_mov_b32 s6, s9
; GFX10-NEXT: s_and_b32 s9, exec_lo, s5
-; GFX10-NEXT: s_mov_b32 s5, s8
-; GFX10-NEXT: s_or_b32 s7, s7, s9
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_or_b32 s7, s7, s10
+; GFX10-NEXT: s_or_b32 s8, s8, s9
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB0_1
; GFX10-NEXT: ; %bb.2: ; %exit
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s7
; GFX10-NEXT: flat_store_dword v[2:3], v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -133,26 +137,30 @@ define void @divergent_i1_xor_used_outside_loop(float %val, float %pre.cond.val,
; GFX10-LABEL: divergent_i1_xor_used_outside_loop:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cmp_lt_f32_e64 s5, 1.0, v1
+; GFX10-NEXT: v_cmp_lt_f32_e64 s6, 1.0, v1
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: s_mov_b32 s6, 0
+; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: ; implicit-def: $sgpr7
+; GFX10-NEXT: ; implicit-def: $sgpr8
; GFX10-NEXT: .LBB2_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s6
-; GFX10-NEXT: s_mov_b32 s8, exec_lo
-; GFX10-NEXT: s_add_i32 s6, s6, 1
-; GFX10-NEXT: s_xor_b32 s8, s5, s8
+; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s4
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_add_i32 s4, s4, 1
+; GFX10-NEXT: s_xor_b32 s9, s6, s9
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v0
-; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_andn2_b32 s8, s8, exec_lo
; GFX10-NEXT: s_andn2_b32 s7, s7, exec_lo
+; GFX10-NEXT: s_and_b32 s10, exec_lo, s6
+; GFX10-NEXT: s_mov_b32 s6, s9
; GFX10-NEXT: s_and_b32 s9, exec_lo, s5
-; GFX10-NEXT: s_mov_b32 s5, s8
-; GFX10-NEXT: s_or_b32 s7, s7, s9
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_or_b32 s7, s7, s10
+; GFX10-NEXT: s_or_b32 s8, s8, s9
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB2_1
; GFX10-NEXT: ; %bb.2: ; %exit
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_cndmask_b32_e64 v0, 1.0, 0, s7
; GFX10-NEXT: flat_store_dword v[2:3], v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -180,28 +188,32 @@ define void @divergent_i1_xor_used_outside_loop_twice(float %val, float %pre.con
; GFX10-LABEL: divergent_i1_xor_used_outside_loop_twice:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cmp_lt_f32_e64 s5, 1.0, v1
+; GFX10-NEXT: v_cmp_lt_f32_e64 s6, 1.0, v1
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: s_mov_b32 s6, 0
+; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: ; implicit-def: $sgpr7
+; GFX10-NEXT: ; implicit-def: $sgpr8
; GFX10-NEXT: .LBB3_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s6
-; GFX10-NEXT: s_mov_b32 s8, exec_lo
-; GFX10-NEXT: s_add_i32 s6, s6, 1
-; GFX10-NEXT: s_xor_b32 s8, s5, s8
+; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s4
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_add_i32 s4, s4, 1
+; GFX10-NEXT: s_xor_b32 s9, s6, s9
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v0
-; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_andn2_b32 s8, s8, exec_lo
+; GFX10-NEXT: s_and_b32 s10, exec_lo, s6
; GFX10-NEXT: s_andn2_b32 s7, s7, exec_lo
+; GFX10-NEXT: s_mov_b32 s6, s9
; GFX10-NEXT: s_and_b32 s9, exec_lo, s5
-; GFX10-NEXT: s_mov_b32 s5, s8
+; GFX10-NEXT: s_or_b32 s8, s8, s10
; GFX10-NEXT: s_or_b32 s7, s7, s9
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB3_1
; GFX10-NEXT: ; %bb.2: ; %exit
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v0, 1.0, 0, s7
-; GFX10-NEXT: v_cndmask_b32_e64 v1, 2.0, -1.0, s7
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s7
+; GFX10-NEXT: v_cndmask_b32_e64 v0, 1.0, 0, s8
+; GFX10-NEXT: v_cndmask_b32_e64 v1, 2.0, -1.0, s8
; GFX10-NEXT: flat_store_dword v[2:3], v0
; GFX10-NEXT: flat_store_dword v[4:5], v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -247,35 +259,42 @@ define void @divergent_i1_xor_used_outside_loop_larger_loop_body(i32 %num.elts,
; GFX10-NEXT: s_cbranch_execz .LBB4_6
; GFX10-NEXT: ; %bb.1: ; %loop.start.preheader
; GFX10-NEXT: s_mov_b32 s8, 0
-; GFX10-NEXT: ; implicit-def: $sgpr10
; GFX10-NEXT: ; implicit-def: $sgpr11
+; GFX10-NEXT: ; implicit-def: $sgpr12
+; GFX10-NEXT: ; implicit-def: $sgpr10
; GFX10-NEXT: ; implicit-def: $sgpr9
; GFX10-NEXT: s_branch .LBB4_3
; GFX10-NEXT: .LBB4_2: ; %Flow
; GFX10-NEXT: ; in Loop: Header=BB4_3 Depth=1
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_xor_b32 s5, s11, exec_lo
-; GFX10-NEXT: s_and_b32 s12, exec_lo, s10
-; GFX10-NEXT: s_or_b32 s8, s12, s8
+; GFX10-NEXT: s_mov_b32 s5, exec_lo
+; GFX10-NEXT: s_xor_b32 s5, s12, s5
+; GFX10-NEXT: s_and_b32 s13, exec_lo, s11
+; GFX10-NEXT: s_or_b32 s13, s13, s8
; GFX10-NEXT: s_andn2_b32 s9, s9, exec_lo
; GFX10-NEXT: s_and_b32 s5, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 s10, s10, exec_lo
+; GFX10-NEXT: s_and_b32 s14, exec_lo, s13
; GFX10-NEXT: s_or_b32 s9, s9, s5
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
+; GFX10-NEXT: s_andn2_b32 s5, s8, exec_lo
+; GFX10-NEXT: s_or_b32 s10, s10, s14
+; GFX10-NEXT: s_or_b32 s8, s5, s14
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s13
; GFX10-NEXT: s_cbranch_execz .LBB4_5
; GFX10-NEXT: .LBB4_3: ; %loop.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_ashr_i32 s5, s4, 31
-; GFX10-NEXT: s_andn2_b32 s10, s10, exec_lo
-; GFX10-NEXT: s_lshl_b64 s[12:13], s[4:5], 2
-; GFX10-NEXT: s_andn2_b32 s5, s11, exec_lo
-; GFX10-NEXT: v_mov_b32_e32 v5, s12
-; GFX10-NEXT: v_mov_b32_e32 v6, s13
-; GFX10-NEXT: s_and_b32 s11, exec_lo, exec_lo
+; GFX10-NEXT: s_andn2_b32 s11, s11, exec_lo
+; GFX10-NEXT: s_lshl_b64 s[14:15], s[4:5], 2
+; GFX10-NEXT: s_andn2_b32 s5, s12, exec_lo
+; GFX10-NEXT: v_mov_b32_e32 v5, s14
+; GFX10-NEXT: v_mov_b32_e32 v6, s15
; GFX10-NEXT: s_and_b32 s12, exec_lo, exec_lo
-; GFX10-NEXT: s_or_b32 s11, s5, s11
+; GFX10-NEXT: s_and_b32 s13, exec_lo, exec_lo
+; GFX10-NEXT: s_or_b32 s12, s5, s12
; GFX10-NEXT: v_add_co_u32 v5, vcc_lo, v1, v5
; GFX10-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, v2, v6, vcc_lo
-; GFX10-NEXT: s_or_b32 s10, s10, s12
+; GFX10-NEXT: s_or_b32 s11, s11, s13
; GFX10-NEXT: global_load_dword v5, v[5:6], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v5
@@ -284,16 +303,16 @@ define void @divergent_i1_xor_used_outside_loop_larger_loop_body(i32 %num.elts,
; GFX10-NEXT: ; %bb.4: ; %loop.cond
; GFX10-NEXT: ; in Loop: Header=BB4_3 Depth=1
; GFX10-NEXT: v_cmp_lt_i32_e32 vcc_lo, s4, v0
+; GFX10-NEXT: s_andn2_b32 s12, s12, exec_lo
+; GFX10-NEXT: s_and_b32 s13, exec_lo, 0
; GFX10-NEXT: s_andn2_b32 s11, s11, exec_lo
-; GFX10-NEXT: s_and_b32 s12, exec_lo, 0
-; GFX10-NEXT: s_andn2_b32 s10, s10, exec_lo
; GFX10-NEXT: s_add_i32 s4, s4, 1
-; GFX10-NEXT: s_and_b32 s13, exec_lo, vcc_lo
-; GFX10-NEXT: s_or_b32 s11, s11, s12
-; GFX10-NEXT: s_or_b32 s10, s10, s13
+; GFX10-NEXT: s_and_b32 s14, exec_lo, vcc_lo
+; GFX10-NEXT: s_or_b32 s12, s12, s13
+; GFX10-NEXT: s_or_b32 s11, s11, s14
; GFX10-NEXT: s_branch .LBB4_2
; GFX10-NEXT: .LBB4_5: ; %loop.exit.guard
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s10
; GFX10-NEXT: s_andn2_b32 s4, s6, exec_lo
; GFX10-NEXT: s_and_b32 s5, exec_lo, s9
; GFX10-NEXT: s_or_b32 s6, s4, s5
@@ -349,13 +368,17 @@ define void @divergent_i1_icmp_used_outside_loop(i32 %v0, i32 %v1, ptr addrspace
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_mov_b32 s6, 0
+; GFX10-NEXT: ; implicit-def: $sgpr8
; GFX10-NEXT: ; implicit-def: $sgpr7
; GFX10-NEXT: s_branch .LBB5_2
; GFX10-NEXT: .LBB5_1: ; %Flow
; GFX10-NEXT: ; in Loop: Header=BB5_2 Depth=1
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX10-NEXT: s_and_b32 s5, exec_lo, s5
; GFX10-NEXT: s_or_b32 s6, s5, s6
+; GFX10-NEXT: s_andn2_b32 s5, s8, exec_lo
+; GFX10-NEXT: s_and_b32 s8, exec_lo, s6
+; GFX10-NEXT: s_or_b32 s8, s5, s8
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX10-NEXT: s_cbranch_execz .LBB5_6
; GFX10-NEXT: .LBB5_2: ; %cond.block.0
@@ -365,7 +388,7 @@ define void @divergent_i1_icmp_used_outside_loop(i32 %v0, i32 %v1, ptr addrspace
; GFX10-NEXT: s_andn2_b32 s5, s7, exec_lo
; GFX10-NEXT: s_and_b32 s7, exec_lo, vcc_lo
; GFX10-NEXT: s_or_b32 s7, s5, s7
-; GFX10-NEXT: s_and_saveexec_b32 s8, vcc_lo
+; GFX10-NEXT: s_and_saveexec_b32 s9, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB5_4
; GFX10-NEXT: ; %bb.3: ; %if.block.0
; GFX10-NEXT: ; in Loop: Header=BB5_2 Depth=1
@@ -380,20 +403,20 @@ define void @divergent_i1_icmp_used_outside_loop(i32 %v0, i32 %v1, ptr addrspace
; GFX10-NEXT: .LBB5_4: ; %loop.break.block
; GFX10-NEXT: ; in Loop: Header=BB5_2 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, s4, v1
; GFX10-NEXT: s_mov_b32 s5, exec_lo
-; GFX10-NEXT: s_and_saveexec_b32 s8, vcc_lo
+; GFX10-NEXT: s_and_saveexec_b32 s9, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB5_1
; GFX10-NEXT: ; %bb.5: ; %loop.cond
; GFX10-NEXT: ; in Loop: Header=BB5_2 Depth=1
; GFX10-NEXT: s_andn2_b32 s5, s5, exec_lo
-; GFX10-NEXT: s_and_b32 s9, exec_lo, 0
+; GFX10-NEXT: s_and_b32 s10, exec_lo, 0
; GFX10-NEXT: s_add_i32 s4, s4, 1
-; GFX10-NEXT: s_or_b32 s5, s5, s9
+; GFX10-NEXT: s_or_b32 s5, s5, s10
; GFX10-NEXT: s_branch .LBB5_1
; GFX10-NEXT: .LBB5_6: ; %cond.block.1
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_and_saveexec_b32 s4, s7
; GFX10-NEXT: s_cbranch_execz .LBB5_8
; GFX10-NEXT: ; %bb.7: ; %if.block.1
@@ -460,49 +483,55 @@ define amdgpu_ps void @divergent_i1_freeze_used_outside_loop(i32 %n, ptr addrspa
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_mov_b32 s1, exec_lo
; GFX10-NEXT: s_mov_b32 s0, 0
-; GFX10-NEXT: s_mov_b32 s2, 0
+; GFX10-NEXT: s_mov_b32 s3, 0
+; GFX10-NEXT: ; implicit-def: $sgpr2
; GFX10-NEXT: ; implicit-def: $sgpr4
-; GFX10-NEXT: ; implicit-def: $sgpr3
+; GFX10-NEXT: ; implicit-def: $sgpr5
; GFX10-NEXT: s_branch .LBB6_2
; GFX10-NEXT: .LBB6_1: ; %loop.cond
; GFX10-NEXT: ; in Loop: Header=BB6_2 Depth=1
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
; GFX10-NEXT: v_cmp_lt_i32_e32 vcc_lo, s0, v0
; GFX10-NEXT: s_add_i32 s0, s0, 1
-; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX10-NEXT: s_or_b32 s6, vcc_lo, s3
+; GFX10-NEXT: s_andn2_b32 s5, s5, exec_lo
+; GFX10-NEXT: s_and_b32 s7, exec_lo, s2
+; GFX10-NEXT: s_andn2_b32 s4, s4, exec_lo
; GFX10-NEXT: s_andn2_b32 s3, s3, exec_lo
-; GFX10-NEXT: s_and_b32 s5, exec_lo, s4
+; GFX10-NEXT: s_and_b32 s8, exec_lo, s6
; GFX10-NEXT: s_andn2_b32 s1, s1, exec_lo
-; GFX10-NEXT: s_or_b32 s3, s3, s5
-; GFX10-NEXT: s_or_b32 s1, s1, s5
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX10-NEXT: s_or_b32 s5, s5, s7
+; GFX10-NEXT: s_or_b32 s4, s4, s8
+; GFX10-NEXT: s_or_b32 s3, s3, s8
+; GFX10-NEXT: s_or_b32 s1, s1, s7
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX10-NEXT: s_cbranch_execz .LBB6_4
; GFX10-NEXT: .LBB6_2: ; %loop.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_andn2_b32 s4, s4, exec_lo
-; GFX10-NEXT: s_and_b32 s5, exec_lo, s1
-; GFX10-NEXT: s_or_b32 s4, s4, s5
-; GFX10-NEXT: s_and_saveexec_b32 s5, s1
+; GFX10-NEXT: s_andn2_b32 s2, s2, exec_lo
+; GFX10-NEXT: s_and_b32 s6, exec_lo, s1
+; GFX10-NEXT: s_or_b32 s2, s2, s6
+; GFX10-NEXT: s_and_saveexec_b32 s6, s1
; GFX10-NEXT: s_cbranch_execz .LBB6_1
; GFX10-NEXT: ; %bb.3: ; %is.eq.zero
; GFX10-NEXT: ; in Loop: Header=BB6_2 Depth=1
; GFX10-NEXT: s_ashr_i32 s1, s0, 31
-; GFX10-NEXT: s_lshl_b64 s[6:7], s[0:1], 2
-; GFX10-NEXT: s_andn2_b32 s1, s4, exec_lo
-; GFX10-NEXT: v_mov_b32_e32 v5, s6
-; GFX10-NEXT: v_mov_b32_e32 v6, s7
+; GFX10-NEXT: s_lshl_b64 s[8:9], s[0:1], 2
+; GFX10-NEXT: s_andn2_b32 s1, s2, exec_lo
+; GFX10-NEXT: v_mov_b32_e32 v5, s8
+; GFX10-NEXT: v_mov_b32_e32 v6, s9
; GFX10-NEXT: v_add_co_u32 v5, vcc_lo, v1, v5
; GFX10-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, v2, v6, vcc_lo
; GFX10-NEXT: global_load_dword v5, v[5:6], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v5
-; GFX10-NEXT: s_and_b32 s4, exec_lo, vcc_lo
-; GFX10-NEXT: s_or_b32 s4, s1, s4
+; GFX10-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; GFX10-NEXT: s_or_b32 s2, s1, s2
; GFX10-NEXT: ; implicit-def: $sgpr1
; GFX10-NEXT: s_branch .LBB6_1
; GFX10-NEXT: .LBB6_4: ; %exit
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s2
-; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s3
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s5
; GFX10-NEXT: flat_store_dword v[3:4], v0
; GFX10-NEXT: s_endpgm
entry:
@@ -539,35 +568,41 @@ define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, ptr addrspace(1) %a
; GFX10-NEXT: s_mov_b32 s0, 0
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: ; implicit-def: $sgpr6
-; GFX10-NEXT: ; implicit-def: $sgpr7
+; GFX10-NEXT: ; implicit-def: $sgpr8
; GFX10-NEXT: ; implicit-def: $sgpr5
+; GFX10-NEXT: ; implicit-def: $sgpr7
; GFX10-NEXT: s_branch .LBB7_2
; GFX10-NEXT: .LBB7_1: ; %Flow
; GFX10-NEXT: ; in Loop: Header=BB7_2 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX10-NEXT: s_and_b32 s1, exec_lo, s6
-; GFX10-NEXT: s_or_b32 s4, s1, s4
-; GFX10-NEXT: s_andn2_b32 s1, s5, exec_lo
-; GFX10-NEXT: s_and_b32 s2, exec_lo, s7
-; GFX10-NEXT: s_or_b32 s5, s1, s2
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_or_b32 s1, s1, s4
+; GFX10-NEXT: s_andn2_b32 s2, s7, exec_lo
+; GFX10-NEXT: s_and_b32 s9, exec_lo, s1
+; GFX10-NEXT: s_andn2_b32 s3, s5, exec_lo
+; GFX10-NEXT: s_and_b32 s5, exec_lo, s8
+; GFX10-NEXT: s_or_b32 s7, s2, s9
+; GFX10-NEXT: s_andn2_b32 s2, s4, exec_lo
+; GFX10-NEXT: s_or_b32 s5, s3, s5
+; GFX10-NEXT: s_or_b32 s4, s2, s9
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s1
; GFX10-NEXT: s_cbranch_execz .LBB7_4
; GFX10-NEXT: .LBB7_2: ; %A
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_ashr_i32 s1, s0, 31
-; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_lshl_b64 s[2:3], s[0:1], 2
-; GFX10-NEXT: s_andn2_b32 s1, s7, exec_lo
+; GFX10-NEXT: s_andn2_b32 s1, s8, exec_lo
; GFX10-NEXT: v_mov_b32_e32 v6, s2
; GFX10-NEXT: v_mov_b32_e32 v7, s3
-; GFX10-NEXT: s_and_b32 s7, exec_lo, s8
+; GFX10-NEXT: s_and_b32 s8, exec_lo, s9
; GFX10-NEXT: s_andn2_b32 s6, s6, exec_lo
-; GFX10-NEXT: s_and_b32 s8, exec_lo, exec_lo
+; GFX10-NEXT: s_and_b32 s9, exec_lo, exec_lo
; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v2, v6
; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v3, v7, vcc_lo
-; GFX10-NEXT: s_or_b32 s7, s1, s7
-; GFX10-NEXT: s_or_b32 s6, s6, s8
+; GFX10-NEXT: s_or_b32 s8, s1, s8
+; GFX10-NEXT: s_or_b32 s6, s6, s9
; GFX10-NEXT: global_load_dword v6, v[6:7], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v6
@@ -582,12 +617,12 @@ define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, ptr addrspace(1) %a
; GFX10-NEXT: s_cselect_b32 s0, exec_lo, 0
; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v0, v6
; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v1, v7, vcc_lo
-; GFX10-NEXT: s_andn2_b32 s3, s7, exec_lo
-; GFX10-NEXT: s_and_b32 s7, exec_lo, 0
+; GFX10-NEXT: s_andn2_b32 s3, s8, exec_lo
+; GFX10-NEXT: s_and_b32 s8, exec_lo, 0
; GFX10-NEXT: s_andn2_b32 s6, s6, exec_lo
; GFX10-NEXT: global_load_dword v8, v[6:7], off
; GFX10-NEXT: s_and_b32 s0, exec_lo, s0
-; GFX10-NEXT: s_or_b32 s7, s3, s7
+; GFX10-NEXT: s_or_b32 s8, s3, s8
; GFX10-NEXT: s_or_b32 s6, s6, s0
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -595,7 +630,7 @@ define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, ptr addrspace(1) %a
; GFX10-NEXT: global_store_dword v[6:7], v8, off
; GFX10-NEXT: s_branch .LBB7_1
; GFX10-NEXT: .LBB7_4: ; %loop.exit.guard
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s7
; GFX10-NEXT: s_and_saveexec_b32 s0, s5
; GFX10-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX10-NEXT: s_cbranch_execz .LBB7_6
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll
index 66af63e968b0d..8367f1b38e106 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll
@@ -109,14 +109,20 @@ define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, ptr addrspace(1) %a
; GFX10-NEXT: s_mov_b32 s0, 0
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: ; implicit-def: $sgpr5
+; GFX10-NEXT: ; implicit-def: $sgpr6
; GFX10-NEXT: s_branch .LBB2_2
; GFX10-NEXT: .LBB2_1: ; %Flow
; GFX10-NEXT: ; in Loop: Header=BB2_2 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX10-NEXT: s_and_b32 s1, exec_lo, s5
-; GFX10-NEXT: s_or_b32 s4, s1, s4
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_or_b32 s1, s1, s4
+; GFX10-NEXT: s_andn2_b32 s2, s6, exec_lo
+; GFX10-NEXT: s_and_b32 s3, exec_lo, s1
+; GFX10-NEXT: s_andn2_b32 s4, s4, exec_lo
+; GFX10-NEXT: s_or_b32 s6, s2, s3
+; GFX10-NEXT: s_or_b32 s4, s4, s3
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s1
; GFX10-NEXT: s_cbranch_execz .LBB2_4
; GFX10-NEXT: .LBB2_2: ; %A
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -183,20 +189,26 @@ define amdgpu_cs void @loop_with_2breaks(ptr addrspace(1) %x, ptr addrspace(1) %
; GFX10-NEXT: s_mov_b32 s0, 0
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: ; implicit-def: $sgpr5
+; GFX10-NEXT: ; implicit-def: $sgpr6
; GFX10-NEXT: s_branch .LBB3_3
; GFX10-NEXT: .LBB3_1: ; %Flow3
; GFX10-NEXT: ; in Loop: Header=BB3_3 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s7
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_andn2_b32 s2, s5, exec_lo
-; GFX10-NEXT: s_and_b32 s3, exec_lo, s6
+; GFX10-NEXT: s_and_b32 s3, exec_lo, s7
; GFX10-NEXT: s_or_b32 s5, s2, s3
; GFX10-NEXT: .LBB3_2: ; %Flow
; GFX10-NEXT: ; in Loop: Header=BB3_3 Depth=1
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX10-NEXT: s_and_b32 s1, exec_lo, s5
-; GFX10-NEXT: s_or_b32 s4, s1, s4
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_or_b32 s1, s1, s4
+; GFX10-NEXT: s_andn2_b32 s2, s6, exec_lo
+; GFX10-NEXT: s_and_b32 s3, exec_lo, s1
+; GFX10-NEXT: s_andn2_b32 s4, s4, exec_lo
+; GFX10-NEXT: s_or_b32 s6, s2, s3
+; GFX10-NEXT: s_or_b32 s4, s4, s3
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s1
; GFX10-NEXT: s_cbranch_execz .LBB3_6
; GFX10-NEXT: .LBB3_3: ; %A
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -218,13 +230,13 @@ define amdgpu_cs void @loop_with_2breaks(ptr addrspace(1) %x, ptr addrspace(1) %
; GFX10-NEXT: ; in Loop: Header=BB3_3 Depth=1
; GFX10-NEXT: v_mov_b32_e32 v6, s2
; GFX10-NEXT: v_mov_b32_e32 v7, s3
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s7, exec_lo
; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v4, v6
; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v5, v7, vcc_lo
; GFX10-NEXT: global_load_dword v6, v[6:7], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v6
-; GFX10-NEXT: s_and_saveexec_b32 s7, vcc_lo
+; GFX10-NEXT: s_and_saveexec_b32 s8, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB3_1
; GFX10-NEXT: ; %bb.5: ; %loop.body
; GFX10-NEXT: ; in Loop: Header=BB3_3 Depth=1
@@ -235,10 +247,10 @@ define amdgpu_cs void @loop_with_2breaks(ptr addrspace(1) %x, ptr addrspace(1) %
; GFX10-NEXT: s_cselect_b32 s0, exec_lo, 0
; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v0, v6
; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v1, v7, vcc_lo
-; GFX10-NEXT: s_andn2_b32 s3, s6, exec_lo
+; GFX10-NEXT: s_andn2_b32 s3, s7, exec_lo
; GFX10-NEXT: s_and_b32 s0, exec_lo, s0
; GFX10-NEXT: global_load_dword v8, v[6:7], off
-; GFX10-NEXT: s_or_b32 s6, s3, s0
+; GFX10-NEXT: s_or_b32 s7, s3, s0
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v8, 1, v8
@@ -281,26 +293,32 @@ define amdgpu_cs void @loop_with_3breaks(ptr addrspace(1) %x, ptr addrspace(1) %
; GFX10-NEXT: s_mov_b32 s0, 0
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: ; implicit-def: $sgpr5
+; GFX10-NEXT: ; implicit-def: $sgpr6
; GFX10-NEXT: s_branch .LBB4_4
; GFX10-NEXT: .LBB4_1: ; %Flow5
; GFX10-NEXT: ; in Loop: Header=BB4_4 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s9
-; GFX10-NEXT: s_andn2_b32 s2, s6, exec_lo
-; GFX10-NEXT: s_and_b32 s3, exec_lo, s8
-; GFX10-NEXT: s_or_b32 s6, s2, s3
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s10
+; GFX10-NEXT: s_andn2_b32 s2, s7, exec_lo
+; GFX10-NEXT: s_and_b32 s3, exec_lo, s9
+; GFX10-NEXT: s_or_b32 s7, s2, s3
; GFX10-NEXT: .LBB4_2: ; %Flow4
; GFX10-NEXT: ; in Loop: Header=BB4_4 Depth=1
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s7
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_andn2_b32 s2, s5, exec_lo
-; GFX10-NEXT: s_and_b32 s3, exec_lo, s6
+; GFX10-NEXT: s_and_b32 s3, exec_lo, s7
; GFX10-NEXT: s_or_b32 s5, s2, s3
; GFX10-NEXT: .LBB4_3: ; %Flow
; GFX10-NEXT: ; in Loop: Header=BB4_4 Depth=1
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX10-NEXT: s_and_b32 s1, exec_lo, s5
-; GFX10-NEXT: s_or_b32 s4, s1, s4
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_or_b32 s1, s1, s4
+; GFX10-NEXT: s_andn2_b32 s2, s6, exec_lo
+; GFX10-NEXT: s_and_b32 s3, exec_lo, s1
+; GFX10-NEXT: s_andn2_b32 s4, s4, exec_lo
+; GFX10-NEXT: s_or_b32 s6, s2, s3
+; GFX10-NEXT: s_or_b32 s4, s4, s3
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s1
; GFX10-NEXT: s_cbranch_execz .LBB4_8
; GFX10-NEXT: .LBB4_4: ; %A
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -322,25 +340,25 @@ define amdgpu_cs void @loop_with_3breaks(ptr addrspace(1) %x, ptr addrspace(1) %
; GFX10-NEXT: ; in Loop: Header=BB4_4 Depth=1
; GFX10-NEXT: v_mov_b32_e32 v8, s2
; GFX10-NEXT: v_mov_b32_e32 v9, s3
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s7, exec_lo
; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v4, v8
; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v5, v9, vcc_lo
; GFX10-NEXT: global_load_dword v8, v[8:9], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
-; GFX10-NEXT: s_and_saveexec_b32 s7, vcc_lo
+; GFX10-NEXT: s_and_saveexec_b32 s8, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB4_2
; GFX10-NEXT: ; %bb.6: ; %C
; GFX10-NEXT: ; in Loop: Header=BB4_4 Depth=1
; GFX10-NEXT: v_mov_b32_e32 v8, s2
; GFX10-NEXT: v_mov_b32_e32 v9, s3
-; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v6, v8
; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v7, v9, vcc_lo
; GFX10-NEXT: global_load_dword v8, v[8:9], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
-; GFX10-NEXT: s_and_saveexec_b32 s9, vcc_lo
+; GFX10-NEXT: s_and_saveexec_b32 s10, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB4_1
; GFX10-NEXT: ; %bb.7: ; %loop.body
; GFX10-NEXT: ; in Loop: Header=BB4_4 Depth=1
@@ -351,10 +369,10 @@ define amdgpu_cs void @loop_with_3breaks(ptr addrspace(1) %x, ptr addrspace(1) %
; GFX10-NEXT: s_cselect_b32 s0, exec_lo, 0
; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v0, v8
; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v1, v9, vcc_lo
-; GFX10-NEXT: s_andn2_b32 s3, s8, exec_lo
+; GFX10-NEXT: s_andn2_b32 s3, s9, exec_lo
; GFX10-NEXT: s_and_b32 s0, exec_lo, s0
; GFX10-NEXT: global_load_dword v10, v[8:9], off
-; GFX10-NEXT: s_or_b32 s8, s3, s0
+; GFX10-NEXT: s_or_b32 s9, s3, s0
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v10, 1, v10
@@ -407,35 +425,41 @@ define amdgpu_cs void @loop_with_div_break_with_body(ptr addrspace(1) %x, ptr ad
; GFX10-NEXT: s_mov_b32 s0, 0
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: ; implicit-def: $sgpr6
-; GFX10-NEXT: ; implicit-def: $sgpr7
+; GFX10-NEXT: ; implicit-def: $sgpr8
; GFX10-NEXT: ; implicit-def: $sgpr5
+; GFX10-NEXT: ; implicit-def: $sgpr7
; GFX10-NEXT: s_branch .LBB5_2
; GFX10-NEXT: .LBB5_1: ; %Flow
; GFX10-NEXT: ; in Loop: Header=BB5_2 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX10-NEXT: s_and_b32 s1, exec_lo, s6
-; GFX10-NEXT: s_or_b32 s4, s1, s4
-; GFX10-NEXT: s_andn2_b32 s1, s5, exec_lo
-; GFX10-NEXT: s_and_b32 s2, exec_lo, s7
-; GFX10-NEXT: s_or_b32 s5, s1, s2
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_or_b32 s1, s1, s4
+; GFX10-NEXT: s_andn2_b32 s2, s7, exec_lo
+; GFX10-NEXT: s_and_b32 s9, exec_lo, s1
+; GFX10-NEXT: s_andn2_b32 s3, s5, exec_lo
+; GFX10-NEXT: s_and_b32 s5, exec_lo, s8
+; GFX10-NEXT: s_or_b32 s7, s2, s9
+; GFX10-NEXT: s_andn2_b32 s2, s4, exec_lo
+; GFX10-NEXT: s_or_b32 s5, s3, s5
+; GFX10-NEXT: s_or_b32 s4, s2, s9
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s1
; GFX10-NEXT: s_cbranch_execz .LBB5_4
; GFX10-NEXT: .LBB5_2: ; %A
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_ashr_i32 s1, s0, 31
-; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_lshl_b64 s[2:3], s[0:1], 2
-; GFX10-NEXT: s_andn2_b32 s1, s7, exec_lo
+; GFX10-NEXT: s_andn2_b32 s1, s8, exec_lo
; GFX10-NEXT: v_mov_b32_e32 v6, s2
; GFX10-NEXT: v_mov_b32_e32 v7, s3
-; GFX10-NEXT: s_and_b32 s7, exec_lo, s8
+; GFX10-NEXT: s_and_b32 s8, exec_lo, s9
; GFX10-NEXT: s_andn2_b32 s6, s6, exec_lo
-; GFX10-NEXT: s_and_b32 s8, exec_lo, exec_lo
+; GFX10-NEXT: s_and_b32 s9, exec_lo, exec_lo
; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v2, v6
; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v3, v7, vcc_lo
-; GFX10-NEXT: s_or_b32 s7, s1, s7
-; GFX10-NEXT: s_or_b32 s6, s6, s8
+; GFX10-NEXT: s_or_b32 s8, s1, s8
+; GFX10-NEXT: s_or_b32 s6, s6, s9
; GFX10-NEXT: global_load_dword v6, v[6:7], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v6
@@ -450,12 +474,12 @@ define amdgpu_cs void @loop_with_div_break_with_body(ptr addrspace(1) %x, ptr ad
; GFX10-NEXT: s_cselect_b32 s0, exec_lo, 0
; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v0, v6
; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v1, v7, vcc_lo
-; GFX10-NEXT: s_andn2_b32 s3, s7, exec_lo
-; GFX10-NEXT: s_and_b32 s7, exec_lo, 0
+; GFX10-NEXT: s_andn2_b32 s3, s8, exec_lo
+; GFX10-NEXT: s_and_b32 s8, exec_lo, 0
; GFX10-NEXT: s_andn2_b32 s6, s6, exec_lo
; GFX10-NEXT: global_load_dword v8, v[6:7], off
; GFX10-NEXT: s_and_b32 s0, exec_lo, s0
-; GFX10-NEXT: s_or_b32 s7, s3, s7
+; GFX10-NEXT: s_or_b32 s8, s3, s8
; GFX10-NEXT: s_or_b32 s6, s6, s0
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -463,7 +487,7 @@ define amdgpu_cs void @loop_with_div_break_with_body(ptr addrspace(1) %x, ptr ad
; GFX10-NEXT: global_store_dword v[6:7], v8, off
; GFX10-NEXT: s_branch .LBB5_1
; GFX10-NEXT: .LBB5_4: ; %loop.exit.guard
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s7
; GFX10-NEXT: s_and_saveexec_b32 s0, s5
; GFX10-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX10-NEXT: s_cbranch_execz .LBB5_6
@@ -526,6 +550,7 @@ define amdgpu_ps i32 @irreducible_cfg(i32 %x, i32 %y, i32 %a0, i32 %a1, i32 %a2,
; GFX10-NEXT: s_mov_b32 s0, exec_lo
; GFX10-NEXT: s_mov_b32 s1, 0
; GFX10-NEXT: s_and_b32 s2, s0, 1
+; GFX10-NEXT: ; implicit-def: $sgpr4
; GFX10-NEXT: s_xor_b32 s0, vcc_lo, s0
; GFX10-NEXT: s_cmp_lg_u32 s2, 0
; GFX10-NEXT: ; implicit-def: $sgpr2
@@ -533,32 +558,35 @@ define amdgpu_ps i32 @irreducible_cfg(i32 %x, i32 %y, i32 %a0, i32 %a1, i32 %a2,
; GFX10-NEXT: s_branch .LBB6_2
; GFX10-NEXT: .LBB6_1: ; %Flow2
; GFX10-NEXT: ; in Loop: Header=BB6_2 Depth=1
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX10-NEXT: s_and_b32 s4, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s7
+; GFX10-NEXT: s_and_b32 s5, exec_lo, s5
; GFX10-NEXT: s_mov_b32 s0, exec_lo
-; GFX10-NEXT: s_or_b32 s1, s4, s1
+; GFX10-NEXT: s_or_b32 s1, s5, s1
+; GFX10-NEXT: s_andn2_b32 s4, s4, exec_lo
+; GFX10-NEXT: s_and_b32 s5, exec_lo, s1
+; GFX10-NEXT: s_or_b32 s4, s4, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s1
; GFX10-NEXT: s_cbranch_execz .LBB6_8
; GFX10-NEXT: .LBB6_2: ; %irr.guard
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB6_6 Depth 2
-; GFX10-NEXT: s_mov_b32 s4, exec_lo
+; GFX10-NEXT: s_mov_b32 s6, exec_lo
; GFX10-NEXT: s_and_saveexec_b32 s5, s0
; GFX10-NEXT: s_xor_b32 s5, exec_lo, s5
; GFX10-NEXT: ; %bb.3: ; %.loopexit
; GFX10-NEXT: ; in Loop: Header=BB6_2 Depth=1
; GFX10-NEXT: v_cmp_gt_i32_e64 s0, v5, v0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
; GFX10-NEXT: s_mov_b32 s7, exec_lo
-; GFX10-NEXT: s_xor_b32 s6, vcc_lo, s6
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_xor_b32 s7, vcc_lo, s7
; GFX10-NEXT: s_andn2_b32 s3, s3, exec_lo
-; GFX10-NEXT: s_or_b32 s6, s0, s6
+; GFX10-NEXT: s_or_b32 s7, s0, s7
; GFX10-NEXT: s_and_b32 s0, exec_lo, s0
-; GFX10-NEXT: s_xor_b32 s6, s6, s7
-; GFX10-NEXT: s_andn2_b32 s4, s4, exec_lo
-; GFX10-NEXT: s_and_b32 s6, exec_lo, s6
+; GFX10-NEXT: s_xor_b32 s7, s7, s8
+; GFX10-NEXT: s_andn2_b32 s6, s6, exec_lo
+; GFX10-NEXT: s_and_b32 s7, exec_lo, s7
; GFX10-NEXT: s_or_b32 s3, s3, s0
-; GFX10-NEXT: s_or_b32 s4, s4, s6
+; GFX10-NEXT: s_or_b32 s6, s6, s7
; GFX10-NEXT: ; %bb.4: ; %Flow1
; GFX10-NEXT: ; in Loop: Header=BB6_2 Depth=1
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
@@ -566,28 +594,32 @@ define amdgpu_ps i32 @irreducible_cfg(i32 %x, i32 %y, i32 %a0, i32 %a1, i32 %a2,
; GFX10-NEXT: s_and_b32 s2, exec_lo, s3
; GFX10-NEXT: s_mov_b32 s5, exec_lo
; GFX10-NEXT: s_or_b32 s2, s0, s2
-; GFX10-NEXT: s_and_saveexec_b32 s6, s4
+; GFX10-NEXT: s_and_saveexec_b32 s7, s6
; GFX10-NEXT: s_cbranch_execz .LBB6_1
; GFX10-NEXT: ; %bb.5: ; %.preheader
; GFX10-NEXT: ; in Loop: Header=BB6_2 Depth=1
; GFX10-NEXT: v_cmp_le_i32_e64 s0, v4, v0
-; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: s_mov_b32 s6, 0
+; GFX10-NEXT: ; implicit-def: $sgpr8
; GFX10-NEXT: .LBB6_6: ; %.inner_loop
; GFX10-NEXT: ; Parent Loop BB6_2 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: s_and_b32 s7, exec_lo, s0
-; GFX10-NEXT: s_or_b32 s4, s7, s4
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_and_b32 s9, exec_lo, s0
+; GFX10-NEXT: s_or_b32 s6, s9, s6
+; GFX10-NEXT: s_andn2_b32 s8, s8, exec_lo
+; GFX10-NEXT: s_and_b32 s9, exec_lo, s6
+; GFX10-NEXT: s_or_b32 s8, s8, s9
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX10-NEXT: s_cbranch_execnz .LBB6_6
; GFX10-NEXT: ; %bb.7: ; %Flow
; GFX10-NEXT: ; in Loop: Header=BB6_2 Depth=1
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_andn2_b32 s0, s5, exec_lo
-; GFX10-NEXT: s_and_b32 s4, exec_lo, 0
-; GFX10-NEXT: s_or_b32 s5, s0, s4
+; GFX10-NEXT: s_and_b32 s5, exec_lo, 0
+; GFX10-NEXT: s_or_b32 s5, s0, s5
; GFX10-NEXT: s_branch .LBB6_1
; GFX10-NEXT: .LBB6_8: ; %.exit
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_cndmask_b32_e64 v0, v2, v3, s2
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll
index ec2607057ecd9..6745df51384fa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll
@@ -9,23 +9,27 @@ define void @temporal_divergent_i1_phi(float %val, ptr %addr) {
; GFX10-NEXT: s_mov_b32 s5, 1
; GFX10-NEXT: s_mov_b32 s6, 0
; GFX10-NEXT: ; implicit-def: $sgpr7
+; GFX10-NEXT: ; implicit-def: $sgpr8
; GFX10-NEXT: .LBB0_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_cvt_f32_u32_e32 v3, s6
-; GFX10-NEXT: s_and_b32 s8, s5, 1
-; GFX10-NEXT: s_cmp_lg_u32 s8, 0
-; GFX10-NEXT: s_cselect_b32 s8, exec_lo, 0
+; GFX10-NEXT: v_cvt_f32_u32_e32 v3, s4
+; GFX10-NEXT: s_and_b32 s9, s5, 1
+; GFX10-NEXT: s_cmp_lg_u32 s9, 0
+; GFX10-NEXT: s_cselect_b32 s9, exec_lo, 0
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v0
; GFX10-NEXT: s_xor_b32 s5, s5, 1
-; GFX10-NEXT: s_add_i32 s6, s6, 1
-; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_add_i32 s4, s4, 1
+; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX10-NEXT: s_andn2_b32 s8, s8, exec_lo
; GFX10-NEXT: s_andn2_b32 s7, s7, exec_lo
-; GFX10-NEXT: s_and_b32 s8, exec_lo, s8
-; GFX10-NEXT: s_or_b32 s7, s7, s8
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_and_b32 s9, exec_lo, s9
+; GFX10-NEXT: s_and_b32 s10, exec_lo, s6
+; GFX10-NEXT: s_or_b32 s7, s7, s9
+; GFX10-NEXT: s_or_b32 s8, s8, s10
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX10-NEXT: s_cbranch_execnz .LBB0_1
; GFX10-NEXT: ; %bb.2: ; %exit
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s7
; GFX10-NEXT: flat_store_dword v[1:2], v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -56,23 +60,27 @@ define void @temporal_divergent_i1_non_phi(float %val, ptr %addr) {
; GFX10-NEXT: s_mov_b32 s5, 1
; GFX10-NEXT: s_mov_b32 s6, 0
; GFX10-NEXT: ; implicit-def: $sgpr7
+; GFX10-NEXT: ; implicit-def: $sgpr8
; GFX10-NEXT: .LBB1_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_cvt_f32_u32_e32 v3, s6
-; GFX10-NEXT: s_and_b32 s8, s5, 1
-; GFX10-NEXT: s_cmp_lg_u32 s8, 0
-; GFX10-NEXT: s_cselect_b32 s8, exec_lo, 0
+; GFX10-NEXT: v_cvt_f32_u32_e32 v3, s4
+; GFX10-NEXT: s_and_b32 s9, s5, 1
+; GFX10-NEXT: s_cmp_lg_u32 s9, 0
+; GFX10-NEXT: s_cselect_b32 s9, exec_lo, 0
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v0
; GFX10-NEXT: s_xor_b32 s5, s5, 1
-; GFX10-NEXT: s_add_i32 s6, s6, 1
-; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_add_i32 s4, s4, 1
+; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX10-NEXT: s_andn2_b32 s8, s8, exec_lo
; GFX10-NEXT: s_andn2_b32 s7, s7, exec_lo
-; GFX10-NEXT: s_and_b32 s8, exec_lo, s8
-; GFX10-NEXT: s_or_b32 s7, s7, s8
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_and_b32 s9, exec_lo, s9
+; GFX10-NEXT: s_and_b32 s10, exec_lo, s6
+; GFX10-NEXT: s_or_b32 s7, s7, s9
+; GFX10-NEXT: s_or_b32 s8, s8, s10
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX10-NEXT: s_cbranch_execnz .LBB1_1
; GFX10-NEXT: ; %bb.2: ; %exit
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_cndmask_b32_e64 v0, 1.0, 0, s7
; GFX10-NEXT: flat_store_dword v[1:2], v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -104,6 +112,7 @@ define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, i32 %x.size, ptr ad
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: ; implicit-def: $sgpr10
+; GFX10-NEXT: ; implicit-def: $sgpr11
; GFX10-NEXT: ; implicit-def: $sgpr9
; GFX10-NEXT: s_branch .LBB2_3
; GFX10-NEXT: .LBB2_1: ; %loop.body
@@ -127,12 +136,17 @@ define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, i32 %x.size, ptr ad
; GFX10-NEXT: s_cmp_lg_u32 s5, 0
; GFX10-NEXT: s_cselect_b32 s5, exec_lo, 0
; GFX10-NEXT: s_and_b32 s6, exec_lo, s10
-; GFX10-NEXT: s_or_b32 s8, s6, s8
-; GFX10-NEXT: s_andn2_b32 s6, s9, exec_lo
+; GFX10-NEXT: s_or_b32 s6, s6, s8
+; GFX10-NEXT: s_andn2_b32 s7, s9, exec_lo
; GFX10-NEXT: s_and_b32 s5, exec_lo, s5
-; GFX10-NEXT: s_or_b32 s9, s6, s5
+; GFX10-NEXT: s_andn2_b32 s11, s11, exec_lo
+; GFX10-NEXT: s_or_b32 s9, s7, s5
+; GFX10-NEXT: s_and_b32 s5, exec_lo, s6
+; GFX10-NEXT: s_andn2_b32 s7, s8, exec_lo
+; GFX10-NEXT: s_or_b32 s11, s11, s5
+; GFX10-NEXT: s_or_b32 s8, s7, s5
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX10-NEXT: s_cbranch_execz .LBB2_5
; GFX10-NEXT: .LBB2_3: ; %A
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -146,8 +160,8 @@ define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, i32 %x.size, ptr ad
; GFX10-NEXT: s_cmp_lg_u32 s5, 0
; GFX10-NEXT: s_cselect_b32 s5, 1, 0
; GFX10-NEXT: s_andn2_b32 s10, s10, exec_lo
-; GFX10-NEXT: s_and_b32 s11, exec_lo, exec_lo
-; GFX10-NEXT: s_or_b32 s10, s10, s11
+; GFX10-NEXT: s_and_b32 s12, exec_lo, exec_lo
+; GFX10-NEXT: s_or_b32 s10, s10, s12
; GFX10-NEXT: s_cmp_lg_u32 s5, 0
; GFX10-NEXT: s_cbranch_scc1 .LBB2_1
; GFX10-NEXT: ; %bb.4: ; in Loop: Header=BB2_3 Depth=1
@@ -155,7 +169,7 @@ define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, i32 %x.size, ptr ad
; GFX10-NEXT: ; implicit-def: $sgpr4
; GFX10-NEXT: s_branch .LBB2_2
; GFX10-NEXT: .LBB2_5: ; %loop.exit.guard
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s11
; GFX10-NEXT: s_and_saveexec_b32 s0, s9
; GFX10-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX10-NEXT: s_cbranch_execz .LBB2_7
@@ -200,52 +214,61 @@ define void @nested_loops_temporal_divergence_inner(float %pre.cond.val, i32 %n.
; GFX10-NEXT: v_cmp_lt_f32_e64 s5, 1.0, v0
; GFX10-NEXT: s_mov_b32 s6, 0
; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: ; implicit-def: $sgpr9
; GFX10-NEXT: .LBB3_1: ; %OuterHeader
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB3_2 Depth 2
; GFX10-NEXT: s_ashr_i32 s7, s6, 31
; GFX10-NEXT: s_mov_b32 s4, s5
; GFX10-NEXT: s_lshl_b64 s[10:11], s[6:7], 2
-; GFX10-NEXT: ; implicit-def: $sgpr9
+; GFX10-NEXT: s_mov_b32 s12, 0
; GFX10-NEXT: v_mov_b32_e32 v6, s10
; GFX10-NEXT: v_mov_b32_e32 v7, s11
-; GFX10-NEXT: s_mov_b32 s10, 0
; GFX10-NEXT: s_mov_b32 s11, 0
+; GFX10-NEXT: ; implicit-def: $sgpr13
+; GFX10-NEXT: ; implicit-def: $sgpr10
; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v2, v6
; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v3, v7, vcc_lo
; GFX10-NEXT: flat_load_dword v0, v[6:7]
; GFX10-NEXT: .LBB3_2: ; %InnerHeader
; GFX10-NEXT: ; Parent Loop BB3_1 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_cvt_f32_u32_e32 v6, s11
-; GFX10-NEXT: s_mov_b32 s12, exec_lo
-; GFX10-NEXT: s_add_i32 s11, s11, 1
-; GFX10-NEXT: s_xor_b32 s4, s4, s12
+; GFX10-NEXT: v_cvt_f32_u32_e32 v6, s12
+; GFX10-NEXT: s_mov_b32 s14, exec_lo
+; GFX10-NEXT: s_add_i32 s12, s12, 1
+; GFX10-NEXT: s_xor_b32 s4, s4, s14
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v0
-; GFX10-NEXT: s_or_b32 s10, vcc_lo, s10
-; GFX10-NEXT: s_andn2_b32 s9, s9, exec_lo
-; GFX10-NEXT: s_and_b32 s12, exec_lo, s4
-; GFX10-NEXT: s_or_b32 s9, s9, s12
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s10
+; GFX10-NEXT: s_or_b32 s11, vcc_lo, s11
+; GFX10-NEXT: s_andn2_b32 s10, s10, exec_lo
+; GFX10-NEXT: s_and_b32 s14, exec_lo, s4
+; GFX10-NEXT: s_andn2_b32 s13, s13, exec_lo
+; GFX10-NEXT: s_and_b32 s15, exec_lo, s11
+; GFX10-NEXT: s_or_b32 s10, s10, s14
+; GFX10-NEXT: s_or_b32 s13, s13, s15
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s11
; GFX10-NEXT: s_cbranch_execnz .LBB3_2
; GFX10-NEXT: ; %bb.3: ; %UseInst
; GFX10-NEXT: ; in Loop: Header=BB3_1 Depth=1
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s10
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s13
; GFX10-NEXT: v_mov_b32_e32 v6, s6
; GFX10-NEXT: v_mov_b32_e32 v7, s7
; GFX10-NEXT: v_cmp_lt_u32_e32 vcc_lo, s6, v1
-; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s9
-; GFX10-NEXT: s_add_i32 s6, s6, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s10
; GFX10-NEXT: v_add_co_u32 v6, s4, v4, v6
; GFX10-NEXT: v_add_co_ci_u32_e64 v7, s4, v5, v7, s4
+; GFX10-NEXT: s_add_i32 s4, s6, 1
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
+; GFX10-NEXT: s_andn2_b32 s6, s9, exec_lo
+; GFX10-NEXT: s_and_b32 s7, exec_lo, s8
; GFX10-NEXT: flat_store_byte v[6:7], v0
+; GFX10-NEXT: s_or_b32 s9, s6, s7
+; GFX10-NEXT: s_mov_b32 s6, s4
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB3_1
; GFX10-NEXT: ; %bb.4: ; %exit
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -291,52 +314,61 @@ define void @nested_loops_temporal_divergence_outer(float %pre.cond.val, i32 %n.
; GFX10-NEXT: v_cmp_lt_f32_e64 s5, 1.0, v0
; GFX10-NEXT: s_mov_b32 s6, 0
; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: ; implicit-def: $sgpr9
; GFX10-NEXT: .LBB4_1: ; %OuterHeader
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB4_2 Depth 2
; GFX10-NEXT: s_ashr_i32 s7, s6, 31
; GFX10-NEXT: s_mov_b32 s4, s5
; GFX10-NEXT: s_lshl_b64 s[10:11], s[6:7], 2
-; GFX10-NEXT: ; implicit-def: $sgpr9
+; GFX10-NEXT: s_mov_b32 s12, 0
; GFX10-NEXT: v_mov_b32_e32 v6, s10
; GFX10-NEXT: v_mov_b32_e32 v7, s11
-; GFX10-NEXT: s_mov_b32 s10, 0
; GFX10-NEXT: s_mov_b32 s11, 0
+; GFX10-NEXT: ; implicit-def: $sgpr13
+; GFX10-NEXT: ; implicit-def: $sgpr10
; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v2, v6
; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v3, v7, vcc_lo
; GFX10-NEXT: flat_load_dword v0, v[6:7]
; GFX10-NEXT: .LBB4_2: ; %InnerHeader
; GFX10-NEXT: ; Parent Loop BB4_1 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_cvt_f32_u32_e32 v6, s11
-; GFX10-NEXT: s_mov_b32 s12, exec_lo
-; GFX10-NEXT: s_add_i32 s11, s11, 1
-; GFX10-NEXT: s_xor_b32 s4, s4, s12
+; GFX10-NEXT: v_cvt_f32_u32_e32 v6, s12
+; GFX10-NEXT: s_mov_b32 s14, exec_lo
+; GFX10-NEXT: s_add_i32 s12, s12, 1
+; GFX10-NEXT: s_xor_b32 s4, s4, s14
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v0
-; GFX10-NEXT: s_or_b32 s10, vcc_lo, s10
-; GFX10-NEXT: s_andn2_b32 s9, s9, exec_lo
-; GFX10-NEXT: s_and_b32 s12, exec_lo, s4
-; GFX10-NEXT: s_or_b32 s9, s9, s12
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s10
+; GFX10-NEXT: s_or_b32 s11, vcc_lo, s11
+; GFX10-NEXT: s_andn2_b32 s10, s10, exec_lo
+; GFX10-NEXT: s_and_b32 s14, exec_lo, s4
+; GFX10-NEXT: s_andn2_b32 s13, s13, exec_lo
+; GFX10-NEXT: s_and_b32 s15, exec_lo, s11
+; GFX10-NEXT: s_or_b32 s10, s10, s14
+; GFX10-NEXT: s_or_b32 s13, s13, s15
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s11
; GFX10-NEXT: s_cbranch_execnz .LBB4_2
; GFX10-NEXT: ; %bb.3: ; %UseInst
; GFX10-NEXT: ; in Loop: Header=BB4_1 Depth=1
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s10
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s13
; GFX10-NEXT: v_mov_b32_e32 v6, s6
; GFX10-NEXT: v_mov_b32_e32 v7, s7
; GFX10-NEXT: v_cmp_lt_u32_e32 vcc_lo, s6, v1
-; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s9
-; GFX10-NEXT: s_add_i32 s6, s6, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s10
; GFX10-NEXT: v_add_co_u32 v6, s4, v4, v6
; GFX10-NEXT: v_add_co_ci_u32_e64 v7, s4, v5, v7, s4
+; GFX10-NEXT: s_add_i32 s4, s6, 1
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
+; GFX10-NEXT: s_andn2_b32 s6, s9, exec_lo
+; GFX10-NEXT: s_and_b32 s7, exec_lo, s8
; GFX10-NEXT: flat_store_byte v[6:7], v0
+; GFX10-NEXT: s_or_b32 s9, s6, s7
+; GFX10-NEXT: s_mov_b32 s6, s4
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB4_1
; GFX10-NEXT: ; %bb.4: ; %exit
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -383,51 +415,60 @@ define void @nested_loops_temporal_divergence_both(float %pre.cond.val, i32 %n.i
; GFX10-NEXT: s_mov_b32 s6, 0
; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: ; implicit-def: $sgpr9
+; GFX10-NEXT: ; implicit-def: $sgpr10
; GFX10-NEXT: .LBB5_1: ; %OuterHeader
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB5_2 Depth 2
; GFX10-NEXT: s_ashr_i32 s7, s6, 31
; GFX10-NEXT: s_mov_b32 s4, s5
-; GFX10-NEXT: s_lshl_b64 s[10:11], s[6:7], 2
-; GFX10-NEXT: v_mov_b32_e32 v8, s10
-; GFX10-NEXT: v_mov_b32_e32 v9, s11
-; GFX10-NEXT: s_mov_b32 s10, 0
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[6:7], 2
; GFX10-NEXT: s_mov_b32 s11, 0
+; GFX10-NEXT: v_mov_b32_e32 v8, s12
+; GFX10-NEXT: v_mov_b32_e32 v9, s13
+; GFX10-NEXT: s_mov_b32 s12, 0
+; GFX10-NEXT: ; implicit-def: $sgpr13
; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v2, v8
; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v3, v9, vcc_lo
; GFX10-NEXT: flat_load_dword v0, v[8:9]
; GFX10-NEXT: .LBB5_2: ; %InnerHeader
; GFX10-NEXT: ; Parent Loop BB5_1 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_cvt_f32_u32_e32 v8, s11
-; GFX10-NEXT: s_mov_b32 s12, exec_lo
-; GFX10-NEXT: s_add_i32 s11, s11, 1
-; GFX10-NEXT: s_xor_b32 s4, s4, s12
+; GFX10-NEXT: v_cvt_f32_u32_e32 v8, s12
+; GFX10-NEXT: s_mov_b32 s14, exec_lo
+; GFX10-NEXT: s_add_i32 s12, s12, 1
+; GFX10-NEXT: s_xor_b32 s4, s4, s14
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v0
-; GFX10-NEXT: s_or_b32 s10, vcc_lo, s10
-; GFX10-NEXT: s_andn2_b32 s9, s9, exec_lo
-; GFX10-NEXT: s_and_b32 s12, exec_lo, s4
-; GFX10-NEXT: s_or_b32 s9, s9, s12
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s10
+; GFX10-NEXT: s_or_b32 s11, vcc_lo, s11
+; GFX10-NEXT: s_andn2_b32 s10, s10, exec_lo
+; GFX10-NEXT: s_and_b32 s14, exec_lo, s4
+; GFX10-NEXT: s_andn2_b32 s13, s13, exec_lo
+; GFX10-NEXT: s_and_b32 s15, exec_lo, s11
+; GFX10-NEXT: s_or_b32 s10, s10, s14
+; GFX10-NEXT: s_or_b32 s13, s13, s15
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s11
; GFX10-NEXT: s_cbranch_execnz .LBB5_2
; GFX10-NEXT: ; %bb.3: ; %UseInst
; GFX10-NEXT: ; in Loop: Header=BB5_1 Depth=1
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s10
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s13
; GFX10-NEXT: v_mov_b32_e32 v8, s6
; GFX10-NEXT: v_mov_b32_e32 v9, s7
; GFX10-NEXT: v_cmp_lt_u32_e32 vcc_lo, s6, v1
-; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s9
-; GFX10-NEXT: s_add_i32 s6, s6, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s10
; GFX10-NEXT: v_add_co_u32 v8, s4, v4, v8
; GFX10-NEXT: v_add_co_ci_u32_e64 v9, s4, v5, v9, s4
+; GFX10-NEXT: s_add_i32 s4, s6, 1
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
+; GFX10-NEXT: s_andn2_b32 s6, s9, exec_lo
+; GFX10-NEXT: s_and_b32 s7, exec_lo, s8
; GFX10-NEXT: flat_store_byte v[8:9], v0
+; GFX10-NEXT: s_or_b32 s9, s6, s7
+; GFX10-NEXT: s_mov_b32 s6, s4
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB5_1
; GFX10-NEXT: ; %bb.4: ; %exit
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX10-NEXT: flat_store_byte v[6:7], v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.ll
index f020c11961abc..3d09730543919 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.ll
@@ -5,19 +5,23 @@ define void @temporal_divergent_i32(float %val, ptr %addr) {
; GFX10-LABEL: temporal_divergent_i32:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, -1
-; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: s_mov_b32 s4, -1
+; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: ; implicit-def: $sgpr6
; GFX10-NEXT: .LBB0_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_add_i32 s5, s5, 1
-; GFX10-NEXT: v_cvt_f32_u32_e32 v3, s5
+; GFX10-NEXT: s_add_i32 s4, s4, 1
+; GFX10-NEXT: v_cvt_f32_u32_e32 v3, s4
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v0
-; GFX10-NEXT: v_mov_b32_e32 v3, s5
-; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_mov_b32_e32 v3, s4
+; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_andn2_b32 s6, s6, exec_lo
+; GFX10-NEXT: s_and_b32 s7, exec_lo, s5
+; GFX10-NEXT: s_or_b32 s6, s6, s7
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB0_1
; GFX10-NEXT: ; %bb.2: ; %exit
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
; GFX10-NEXT: flat_store_dword v[1:2], v3
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -40,19 +44,23 @@ define void @temporal_divergent_i32_multiple_use(float %val, ptr %addr, ptr %add
; GFX10-LABEL: temporal_divergent_i32_multiple_use:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, -1
-; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: s_mov_b32 s4, -1
+; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: ; implicit-def: $sgpr6
; GFX10-NEXT: .LBB1_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_add_i32 s5, s5, 1
-; GFX10-NEXT: v_cvt_f32_u32_e32 v5, s5
+; GFX10-NEXT: s_add_i32 s4, s4, 1
+; GFX10-NEXT: v_cvt_f32_u32_e32 v5, s4
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v0
-; GFX10-NEXT: v_mov_b32_e32 v5, s5
-; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_mov_b32_e32 v5, s4
+; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_andn2_b32 s6, s6, exec_lo
+; GFX10-NEXT: s_and_b32 s7, exec_lo, s5
+; GFX10-NEXT: s_or_b32 s6, s6, s7
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB1_1
; GFX10-NEXT: ; %bb.2: ; %exit
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
; GFX10-NEXT: flat_store_dword v[1:2], v5
; GFX10-NEXT: flat_store_dword v[3:4], v5
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergent-control-flow.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergent-control-flow.ll
index 5b5273daa3cf4..80a2d07aeb329 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergent-control-flow.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergent-control-flow.ll
@@ -210,26 +210,32 @@ define amdgpu_kernel void @break_loop(i32 %arg) {
; CHECK: ; %bb.0: ; %bb
; CHECK-NEXT: s_load_dword s0, s[8:9], 0x0
; CHECK-NEXT: ; implicit-def: $sgpr2_sgpr3
-; CHECK-NEXT: ; implicit-def: $sgpr4
+; CHECK-NEXT: ; implicit-def: $sgpr4_sgpr5
+; CHECK-NEXT: ; implicit-def: $sgpr6
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: v_subrev_u32_e32 v0, s0, v0
; CHECK-NEXT: s_mov_b64 s[0:1], 0
; CHECK-NEXT: s_branch .LBB5_2
; CHECK-NEXT: .LBB5_1: ; %Flow
; CHECK-NEXT: ; in Loop: Header=BB5_2 Depth=1
-; CHECK-NEXT: s_and_b64 s[6:7], exec, s[2:3]
-; CHECK-NEXT: s_or_b64 s[0:1], s[6:7], s[0:1]
-; CHECK-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; CHECK-NEXT: s_and_b64 s[8:9], exec, s[2:3]
+; CHECK-NEXT: s_or_b64 s[8:9], s[8:9], s[0:1]
+; CHECK-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; CHECK-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; CHECK-NEXT: s_andn2_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_or_b64 s[4:5], s[4:5], s[10:11]
+; CHECK-NEXT: s_or_b64 s[0:1], s[0:1], s[10:11]
+; CHECK-NEXT: s_andn2_b64 exec, exec, s[8:9]
; CHECK-NEXT: s_cbranch_execz .LBB5_4
; CHECK-NEXT: .LBB5_2: ; %bb1
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: s_add_i32 s4, s4, 1
-; CHECK-NEXT: s_cmp_ge_i32 s4, 0
-; CHECK-NEXT: s_cselect_b32 s5, 1, 0
+; CHECK-NEXT: s_add_i32 s6, s6, 1
+; CHECK-NEXT: s_cmp_ge_i32 s6, 0
+; CHECK-NEXT: s_cselect_b32 s7, 1, 0
; CHECK-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
-; CHECK-NEXT: s_and_b64 s[6:7], exec, exec
-; CHECK-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
-; CHECK-NEXT: s_cmp_lg_u32 s5, 0
+; CHECK-NEXT: s_and_b64 s[8:9], exec, exec
+; CHECK-NEXT: s_or_b64 s[2:3], s[2:3], s[8:9]
+; CHECK-NEXT: s_cmp_lg_u32 s7, 0
; CHECK-NEXT: s_cbranch_scc1 .LBB5_1
; CHECK-NEXT: ; %bb.3: ; %bb4
; CHECK-NEXT: ; in Loop: Header=BB5_2 Depth=1
@@ -237,8 +243,8 @@ define amdgpu_kernel void @break_loop(i32 %arg) {
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
; CHECK-NEXT: v_cmp_ge_i32_e32 vcc, v0, v1
-; CHECK-NEXT: s_and_b64 s[6:7], exec, vcc
-; CHECK-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
+; CHECK-NEXT: s_and_b64 s[8:9], exec, vcc
+; CHECK-NEXT: s_or_b64 s[2:3], s[2:3], s[8:9]
; CHECK-NEXT: s_branch .LBB5_1
; CHECK-NEXT: .LBB5_4: ; %bb9
; CHECK-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f64.ll
index 9ccdf2f681f4c..dc4a89352dff7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f64.ll
@@ -15,23 +15,32 @@ define amdgpu_ps void @global_atomic_fadd_f64_no_rtn_atomicrmw(ptr addrspace(1)
; GFX90A-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; GFX90A-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
; GFX90A-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64_align2 = GLOBAL_LOAD_DWORDX2 [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (f64) from %ir.ptr, addrspace 1)
+ ; GFX90A-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
; GFX90A-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.2.atomicrmw.start:
; GFX90A-NEXT: successors: %bb.3(0x04000000), %bb.2(0x7c000000)
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: [[PHI:%[0-9]+]]:sreg_64_xexec = PHI %23, %bb.2, [[S_MOV_B64_]], %bb.1
- ; GFX90A-NEXT: [[PHI1:%[0-9]+]]:vreg_64_align2 = PHI [[GLOBAL_LOAD_DWORDX2_]], %bb.1, %21, %bb.2
- ; GFX90A-NEXT: [[V_ADD_F64_e64_:%[0-9]+]]:vreg_64_align2 = nofpexcept V_ADD_F64_e64 0, [[PHI1]], 0, [[REG_SEQUENCE1]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[V_ADD_F64_e64_]], %subreg.sub0_sub1, [[PHI1]], %subreg.sub2_sub3
+ ; GFX90A-NEXT: [[PHI:%[0-9]+]]:sreg_64 = PHI [[S_MOV_B64_]], %bb.1, %31, %bb.2
+ ; GFX90A-NEXT: [[PHI1:%[0-9]+]]:sreg_64 = PHI [[DEF]], %bb.1, %23, %bb.2
+ ; GFX90A-NEXT: [[PHI2:%[0-9]+]]:vreg_64_align2 = PHI [[GLOBAL_LOAD_DWORDX2_]], %bb.1, %21, %bb.2
+ ; GFX90A-NEXT: [[V_ADD_F64_e64_:%[0-9]+]]:vreg_64_align2 = nofpexcept V_ADD_F64_e64 0, [[PHI2]], 0, [[REG_SEQUENCE1]], 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[V_ADD_F64_e64_]], %subreg.sub0_sub1, [[PHI2]], %subreg.sub2_sub3
; GFX90A-NEXT: [[GLOBAL_ATOMIC_CMPSWAP_X2_RTN:%[0-9]+]]:vreg_64_align2 = GLOBAL_ATOMIC_CMPSWAP_X2_RTN [[REG_SEQUENCE]], [[REG_SEQUENCE2]], 0, 1, implicit $exec :: (load store syncscope("wavefront") monotonic monotonic (i64) on %ir.ptr, addrspace 1)
- ; GFX90A-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[GLOBAL_ATOMIC_CMPSWAP_X2_RTN]], [[PHI1]], implicit $exec
- ; GFX90A-NEXT: [[SI_IF_BREAK:%[0-9]+]]:sreg_64_xexec = SI_IF_BREAK [[V_CMP_EQ_U64_e64_]], [[PHI]], implicit-def $scc
+ ; GFX90A-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[GLOBAL_ATOMIC_CMPSWAP_X2_RTN]], [[PHI2]], implicit $exec
+ ; GFX90A-NEXT: [[COPY4:%[0-9]+]]:sreg_64_xexec = COPY [[PHI]]
+ ; GFX90A-NEXT: [[SI_IF_BREAK:%[0-9]+]]:sreg_64_xexec = SI_IF_BREAK [[V_CMP_EQ_U64_e64_]], [[COPY4]], implicit-def $scc
+ ; GFX90A-NEXT: [[COPY5:%[0-9]+]]:sreg_64 = COPY [[SI_IF_BREAK]]
+ ; GFX90A-NEXT: [[S_ANDN2_B64_:%[0-9]+]]:sreg_64 = S_ANDN2_B64 [[PHI1]], $exec, implicit-def $scc
+ ; GFX90A-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 $exec, [[COPY5]], implicit-def $scc
+ ; GFX90A-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_ANDN2_B64_]], [[S_AND_B64_]], implicit-def $scc
+ ; GFX90A-NEXT: [[COPY6:%[0-9]+]]:sreg_64 = COPY [[SI_IF_BREAK]]
; GFX90A-NEXT: SI_LOOP [[SI_IF_BREAK]], %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: S_BRANCH %bb.3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.3.atomicrmw.end:
- ; GFX90A-NEXT: SI_END_CF [[SI_IF_BREAK]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: [[COPY7:%[0-9]+]]:sreg_64_xexec = COPY [[S_OR_B64_]]
+ ; GFX90A-NEXT: SI_END_CF [[COPY7]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: global_atomic_fadd_f64_no_rtn_atomicrmw
@@ -63,28 +72,37 @@ define amdgpu_ps double @global_atomic_fadd_f64_rtn_atomicrmw(ptr addrspace(1) %
; GFX90A-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; GFX90A-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
; GFX90A-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64_align2 = GLOBAL_LOAD_DWORDX2 [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (f64) from %ir.ptr, addrspace 1)
+ ; GFX90A-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
; GFX90A-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.2.atomicrmw.start:
; GFX90A-NEXT: successors: %bb.3(0x04000000), %bb.2(0x7c000000)
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: [[PHI:%[0-9]+]]:sreg_64_xexec = PHI %27, %bb.2, [[S_MOV_B64_]], %bb.1
- ; GFX90A-NEXT: [[PHI1:%[0-9]+]]:vreg_64_align2 = PHI [[GLOBAL_LOAD_DWORDX2_]], %bb.1, %25, %bb.2
- ; GFX90A-NEXT: [[V_ADD_F64_e64_:%[0-9]+]]:vreg_64_align2 = nofpexcept V_ADD_F64_e64 0, [[PHI1]], 0, [[REG_SEQUENCE1]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[V_ADD_F64_e64_]], %subreg.sub0_sub1, [[PHI1]], %subreg.sub2_sub3
+ ; GFX90A-NEXT: [[PHI:%[0-9]+]]:sreg_64 = PHI [[S_MOV_B64_]], %bb.1, %35, %bb.2
+ ; GFX90A-NEXT: [[PHI1:%[0-9]+]]:sreg_64 = PHI [[DEF]], %bb.1, %27, %bb.2
+ ; GFX90A-NEXT: [[PHI2:%[0-9]+]]:vreg_64_align2 = PHI [[GLOBAL_LOAD_DWORDX2_]], %bb.1, %25, %bb.2
+ ; GFX90A-NEXT: [[V_ADD_F64_e64_:%[0-9]+]]:vreg_64_align2 = nofpexcept V_ADD_F64_e64 0, [[PHI2]], 0, [[REG_SEQUENCE1]], 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[V_ADD_F64_e64_]], %subreg.sub0_sub1, [[PHI2]], %subreg.sub2_sub3
; GFX90A-NEXT: [[GLOBAL_ATOMIC_CMPSWAP_X2_RTN:%[0-9]+]]:vreg_64_align2 = GLOBAL_ATOMIC_CMPSWAP_X2_RTN [[REG_SEQUENCE]], [[REG_SEQUENCE2]], 0, 1, implicit $exec :: (load store syncscope("wavefront") monotonic monotonic (i64) on %ir.ptr, addrspace 1)
- ; GFX90A-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[GLOBAL_ATOMIC_CMPSWAP_X2_RTN]], [[PHI1]], implicit $exec
- ; GFX90A-NEXT: [[SI_IF_BREAK:%[0-9]+]]:sreg_64_xexec = SI_IF_BREAK [[V_CMP_EQ_U64_e64_]], [[PHI]], implicit-def $scc
+ ; GFX90A-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[GLOBAL_ATOMIC_CMPSWAP_X2_RTN]], [[PHI2]], implicit $exec
+ ; GFX90A-NEXT: [[COPY4:%[0-9]+]]:sreg_64_xexec = COPY [[PHI]]
+ ; GFX90A-NEXT: [[SI_IF_BREAK:%[0-9]+]]:sreg_64_xexec = SI_IF_BREAK [[V_CMP_EQ_U64_e64_]], [[COPY4]], implicit-def $scc
+ ; GFX90A-NEXT: [[COPY5:%[0-9]+]]:sreg_64 = COPY [[SI_IF_BREAK]]
+ ; GFX90A-NEXT: [[S_ANDN2_B64_:%[0-9]+]]:sreg_64 = S_ANDN2_B64 [[PHI1]], $exec, implicit-def $scc
+ ; GFX90A-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 $exec, [[COPY5]], implicit-def $scc
+ ; GFX90A-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_ANDN2_B64_]], [[S_AND_B64_]], implicit-def $scc
+ ; GFX90A-NEXT: [[COPY6:%[0-9]+]]:sreg_64 = COPY [[SI_IF_BREAK]]
; GFX90A-NEXT: SI_LOOP [[SI_IF_BREAK]], %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: S_BRANCH %bb.3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.3.atomicrmw.end:
- ; GFX90A-NEXT: SI_END_CF [[SI_IF_BREAK]], implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX90A-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_ATOMIC_CMPSWAP_X2_RTN]].sub0
- ; GFX90A-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_ATOMIC_CMPSWAP_X2_RTN]].sub1
- ; GFX90A-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+ ; GFX90A-NEXT: [[COPY7:%[0-9]+]]:sreg_64_xexec = COPY [[S_OR_B64_]]
+ ; GFX90A-NEXT: SI_END_CF [[COPY7]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_ATOMIC_CMPSWAP_X2_RTN]].sub0
+ ; GFX90A-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_ATOMIC_CMPSWAP_X2_RTN]].sub1
+ ; GFX90A-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
; GFX90A-NEXT: $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
- ; GFX90A-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+ ; GFX90A-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
; GFX90A-NEXT: $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
; GFX90A-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-atomicrmw.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-atomicrmw.ll
index 2a47586b84a2b..892a455bf6f6c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-atomicrmw.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-atomicrmw.ll
@@ -23,27 +23,27 @@ define float @test_atomicrmw_fsub(ptr addrspace(3) %addr) {
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
; CHECK-NEXT: [[C:%[0-9]+]]:_(f32) = G_FCONSTANT float 1.000000e+00
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i1) = G_CONSTANT i1 false
; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(f32) = G_LOAD [[COPY]](p3) :: (load (f32) from %ir.addr, addrspace 3)
; CHECK-NEXT: G_BR %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.atomicrmw.start:
; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:_(i64) = G_PHI %19(i64), %bb.2, [[C1]](i64), %bb.1
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:_(i1) = G_PHI %19(i1), %bb.2, [[C1]](i1), %bb.1
; CHECK-NEXT: [[PHI1:%[0-9]+]]:_(f32) = G_PHI [[LOAD]](f32), %bb.1, %18(f32), %bb.2
; CHECK-NEXT: [[FSUB:%[0-9]+]]:_(f32) = G_FSUB [[PHI1]], [[C]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[FSUB]](f32)
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[PHI1]](f32)
; CHECK-NEXT: [[ATOMIC_CMPXCHG_WITH_SUCCESS:%[0-9]+]]:_(i32), [[ATOMIC_CMPXCHG_WITH_SUCCESS1:%[0-9]+]]:_(i1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[COPY]](p3), [[BITCAST1]], [[BITCAST]] :: (load store seq_cst seq_cst (i32) on %ir.addr, addrspace 3)
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[ATOMIC_CMPXCHG_WITH_SUCCESS]](i32)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:_(i64) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[ATOMIC_CMPXCHG_WITH_SUCCESS1]](i1), [[PHI]](i64)
- ; CHECK-NEXT: [[INT1:%[0-9]+]]:_(i1) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.loop), [[INT]](i64)
+ ; CHECK-NEXT: [[INT:%[0-9]+]]:_(i1) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[ATOMIC_CMPXCHG_WITH_SUCCESS1]](i1), [[PHI]](i1)
+ ; CHECK-NEXT: [[INT1:%[0-9]+]]:_(i1) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.loop), [[INT]](i1)
; CHECK-NEXT: G_BRCOND [[INT1]](i1), %bb.3
; CHECK-NEXT: G_BR %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3.atomicrmw.end:
- ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](i64)
+ ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](i1)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST2]](f32)
; CHECK-NEXT: SI_RETURN implicit $vgpr0
%oldval = atomicrmw fsub ptr addrspace(3) %addr, float 1.0 seq_cst
@@ -74,27 +74,27 @@ define <2 x half> @test_atomicrmw_fsub_vector(ptr addrspace(3) %addr) {
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
; CHECK-NEXT: [[C:%[0-9]+]]:_(f16) = G_FCONSTANT half 1.000000e+00
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[C]](f16), [[C]](f16)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i1) = G_CONSTANT i1 false
; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<2 x f16>) = G_LOAD [[COPY]](p3) :: (load (<2 x f16>) from %ir.addr, addrspace 3)
; CHECK-NEXT: G_BR %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.atomicrmw.start:
; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:_(i64) = G_PHI %20(i64), %bb.2, [[C1]](i64), %bb.1
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:_(i1) = G_PHI %20(i1), %bb.2, [[C1]](i1), %bb.1
; CHECK-NEXT: [[PHI1:%[0-9]+]]:_(<2 x f16>) = G_PHI [[LOAD]](<2 x f16>), %bb.1, %19(<2 x f16>), %bb.2
; CHECK-NEXT: [[FSUB:%[0-9]+]]:_(<2 x f16>) = G_FSUB [[PHI1]], [[BUILD_VECTOR]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[FSUB]](<2 x f16>)
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[PHI1]](<2 x f16>)
; CHECK-NEXT: [[ATOMIC_CMPXCHG_WITH_SUCCESS:%[0-9]+]]:_(i32), [[ATOMIC_CMPXCHG_WITH_SUCCESS1:%[0-9]+]]:_(i1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[COPY]](p3), [[BITCAST1]], [[BITCAST]] :: (load store seq_cst seq_cst (i32) on %ir.addr, addrspace 3)
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[ATOMIC_CMPXCHG_WITH_SUCCESS]](i32)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:_(i64) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[ATOMIC_CMPXCHG_WITH_SUCCESS1]](i1), [[PHI]](i64)
- ; CHECK-NEXT: [[INT1:%[0-9]+]]:_(i1) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.loop), [[INT]](i64)
+ ; CHECK-NEXT: [[INT:%[0-9]+]]:_(i1) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[ATOMIC_CMPXCHG_WITH_SUCCESS1]](i1), [[PHI]](i1)
+ ; CHECK-NEXT: [[INT1:%[0-9]+]]:_(i1) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.loop), [[INT]](i1)
; CHECK-NEXT: G_BRCOND [[INT1]](i1), %bb.3
; CHECK-NEXT: G_BR %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3.atomicrmw.end:
- ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](i64)
+ ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](i1)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x f16>)
; CHECK-NEXT: SI_RETURN implicit $vgpr0
%oldval = atomicrmw fsub ptr addrspace(3) %addr, <2 x half> <half 1.0, half 1.0> seq_cst
@@ -110,27 +110,27 @@ define <2 x half> @test_atomicrmw_fmin_vector(ptr addrspace(3) %addr) {
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
; CHECK-NEXT: [[C:%[0-9]+]]:_(f16) = G_FCONSTANT half 1.000000e+00
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[C]](f16), [[C]](f16)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i1) = G_CONSTANT i1 false
; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<2 x f16>) = G_LOAD [[COPY]](p3) :: (load (<2 x f16>) from %ir.addr, addrspace 3)
; CHECK-NEXT: G_BR %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.atomicrmw.start:
; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:_(i64) = G_PHI %20(i64), %bb.2, [[C1]](i64), %bb.1
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:_(i1) = G_PHI %20(i1), %bb.2, [[C1]](i1), %bb.1
; CHECK-NEXT: [[PHI1:%[0-9]+]]:_(<2 x f16>) = G_PHI [[LOAD]](<2 x f16>), %bb.1, %19(<2 x f16>), %bb.2
; CHECK-NEXT: [[FMINNUM:%[0-9]+]]:_(<2 x f16>) = G_FMINNUM [[PHI1]], [[BUILD_VECTOR]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[FMINNUM]](<2 x f16>)
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[PHI1]](<2 x f16>)
; CHECK-NEXT: [[ATOMIC_CMPXCHG_WITH_SUCCESS:%[0-9]+]]:_(i32), [[ATOMIC_CMPXCHG_WITH_SUCCESS1:%[0-9]+]]:_(i1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[COPY]](p3), [[BITCAST1]], [[BITCAST]] :: (load store seq_cst seq_cst (i32) on %ir.addr, addrspace 3)
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[ATOMIC_CMPXCHG_WITH_SUCCESS]](i32)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:_(i64) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[ATOMIC_CMPXCHG_WITH_SUCCESS1]](i1), [[PHI]](i64)
- ; CHECK-NEXT: [[INT1:%[0-9]+]]:_(i1) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.loop), [[INT]](i64)
+ ; CHECK-NEXT: [[INT:%[0-9]+]]:_(i1) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[ATOMIC_CMPXCHG_WITH_SUCCESS1]](i1), [[PHI]](i1)
+ ; CHECK-NEXT: [[INT1:%[0-9]+]]:_(i1) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.loop), [[INT]](i1)
; CHECK-NEXT: G_BRCOND [[INT1]](i1), %bb.3
; CHECK-NEXT: G_BR %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3.atomicrmw.end:
- ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](i64)
+ ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](i1)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x f16>)
; CHECK-NEXT: SI_RETURN implicit $vgpr0
%oldval = atomicrmw fmin ptr addrspace(3) %addr, <2 x half> <half 1.0, half 1.0> seq_cst
@@ -146,27 +146,27 @@ define <2 x half> @test_atomicrmw_fmax_vector(ptr addrspace(3) %addr) {
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
; CHECK-NEXT: [[C:%[0-9]+]]:_(f16) = G_FCONSTANT half 1.000000e+00
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[C]](f16), [[C]](f16)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i1) = G_CONSTANT i1 false
; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<2 x f16>) = G_LOAD [[COPY]](p3) :: (load (<2 x f16>) from %ir.addr, addrspace 3)
; CHECK-NEXT: G_BR %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.atomicrmw.start:
; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:_(i64) = G_PHI %20(i64), %bb.2, [[C1]](i64), %bb.1
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:_(i1) = G_PHI %20(i1), %bb.2, [[C1]](i1), %bb.1
; CHECK-NEXT: [[PHI1:%[0-9]+]]:_(<2 x f16>) = G_PHI [[LOAD]](<2 x f16>), %bb.1, %19(<2 x f16>), %bb.2
; CHECK-NEXT: [[FMAXNUM:%[0-9]+]]:_(<2 x f16>) = G_FMAXNUM [[PHI1]], [[BUILD_VECTOR]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[FMAXNUM]](<2 x f16>)
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[PHI1]](<2 x f16>)
; CHECK-NEXT: [[ATOMIC_CMPXCHG_WITH_SUCCESS:%[0-9]+]]:_(i32), [[ATOMIC_CMPXCHG_WITH_SUCCESS1:%[0-9]+]]:_(i1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[COPY]](p3), [[BITCAST1]], [[BITCAST]] :: (load store seq_cst seq_cst (i32) on %ir.addr, addrspace 3)
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[ATOMIC_CMPXCHG_WITH_SUCCESS]](i32)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:_(i64) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[ATOMIC_CMPXCHG_WITH_SUCCESS1]](i1), [[PHI]](i64)
- ; CHECK-NEXT: [[INT1:%[0-9]+]]:_(i1) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.loop), [[INT]](i64)
+ ; CHECK-NEXT: [[INT:%[0-9]+]]:_(i1) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[ATOMIC_CMPXCHG_WITH_SUCCESS1]](i1), [[PHI]](i1)
+ ; CHECK-NEXT: [[INT1:%[0-9]+]]:_(i1) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.loop), [[INT]](i1)
; CHECK-NEXT: G_BRCOND [[INT1]](i1), %bb.3
; CHECK-NEXT: G_BR %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3.atomicrmw.end:
- ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](i64)
+ ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](i1)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x f16>)
; CHECK-NEXT: SI_RETURN implicit $vgpr0
%oldval = atomicrmw fmax ptr addrspace(3) %addr, <2 x half> <half 1.0, half 1.0> seq_cst
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll
index f9ee0e3cfc25a..9a271b1d09665 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll
@@ -97,7 +97,7 @@ define void @i1_arg_i1_use(i1 %arg) #0 {
; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; CHECK-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; CHECK-NEXT: [[XOR:%[0-9]+]]:_(i1) = G_XOR [[TRUNC]], [[C]]
- ; CHECK-NEXT: [[INT:%[0-9]+]]:_(i1), [[INT1:%[0-9]+]]:_(i64) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.if), [[XOR]](i1)
+ ; CHECK-NEXT: [[INT:%[0-9]+]]:_(i1), [[INT1:%[0-9]+]]:_(i1) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.if), [[XOR]](i1)
; CHECK-NEXT: G_BRCOND [[INT]](i1), %bb.2
; CHECK-NEXT: G_BR %bb.3
; CHECK-NEXT: {{ $}}
@@ -108,7 +108,7 @@ define void @i1_arg_i1_use(i1 %arg) #0 {
; CHECK-NEXT: G_BR %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3.bb2:
- ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT1]](i64)
+ ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT1]](i1)
; CHECK-NEXT: SI_RETURN
bb:
br i1 %arg, label %bb2, label %bb1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i32.ll
index 3b2b817ced498..7f801ff11bcc3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i32.ll
@@ -2,7 +2,7 @@
; RUN: llc -global-isel -mtriple=amdgcn--amdhsa -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgcn--amdhsa -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX11 %s
-define amdgpu_kernel void @test_wave32(i32 %arg0, [8 x i32], i32 %saved) {
+define amdgpu_kernel void @test_wave32(i32 %arg0, [8 x i32], i1 %saved) {
; GFX10-LABEL: test_wave32:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_load_dword s0, s[8:9], 0x0
@@ -16,6 +16,8 @@ define amdgpu_kernel void @test_wave32(i32 %arg0, [8 x i32], i32 %saved) {
; GFX10-NEXT: .LBB0_2: ; %bb
; GFX10-NEXT: s_load_dword s0, s[8:9], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_cmp_lg_u32 s0, 0
+; GFX10-NEXT: s_cselect_b32 s0, exec_lo, 0
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
@@ -36,6 +38,8 @@ define amdgpu_kernel void @test_wave32(i32 %arg0, [8 x i32], i32 %saved) {
; GFX11-NEXT: .LBB0_2: ; %bb
; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s0, 0
+; GFX11-NEXT: s_cselect_b32 s0, exec_lo, 0
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: global_store_b32 v[0:1], v0, off dlc
@@ -50,9 +54,9 @@ mid:
br label %bb
bb:
- call void @llvm.amdgcn.end.cf.i32(i32 %saved)
+ call void @llvm.amdgcn.end.cf(i1 %saved)
store volatile i32 0, ptr addrspace(1) poison
ret void
}
-declare void @llvm.amdgcn.end.cf.i32(i32 %val)
+declare void @llvm.amdgcn.end.cf(i1 %val)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i64.ll
index 3a0ef12139063..b874ffdf5b6a1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i64.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -mtriple=amdgcn--amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefix=GCN %s
-define amdgpu_kernel void @test_wave64(i32 %arg0, i64 %saved) {
+define amdgpu_kernel void @test_wave64(i32 %arg0, i1 %saved) {
; GCN-LABEL: test_wave64:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s0, s[8:9], 0x0
+; GCN-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_cmp_lg_u32 s0, 0
; GCN-NEXT: s_cbranch_scc1 .LBB0_2
@@ -13,8 +13,8 @@ define amdgpu_kernel void @test_wave64(i32 %arg0, i64 %saved) {
; GCN-NEXT: global_store_dword v[0:1], v0, off
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: .LBB0_2: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x8
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_cmp_lg_u32 s1, 0
+; GCN-NEXT: s_cselect_b64 s[0:1], exec, 0
; GCN-NEXT: s_or_b64 exec, exec, s[0:1]
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: global_store_dword v[0:1], v0, off
@@ -29,9 +29,9 @@ mid:
br label %bb
bb:
- call void @llvm.amdgcn.end.cf.i64(i64 %saved)
+ call void @llvm.amdgcn.end.cf(i1 %saved)
store volatile i32 0, ptr addrspace(1) poison
ret void
}
-declare void @llvm.amdgcn.end.cf.i64(i64 %val)
+declare void @llvm.amdgcn.end.cf(i1 %val)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i32.ll
index 4b8e69df76de4..bbdba1ff5e3a1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i32.ll
@@ -10,10 +10,11 @@ define amdgpu_kernel void @test_wave32(i32 %arg0, [8 x i32], i32 %saved) {
; GFX10-NEXT: s_load_dword s1, s[8:9], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_cmp_eq_u32 s0, 0
+; GFX10-NEXT: v_cmp_gt_u32_e32 vcc_lo, s1, v0
; GFX10-NEXT: s_cselect_b32 s0, exec_lo, 0
; GFX10-NEXT: s_and_b32 s0, exec_lo, s0
-; GFX10-NEXT: s_or_b32 s0, s0, s1
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: s_or_b32 s0, s0, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
; GFX10-NEXT: global_store_dword v[0:1], v0, off
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: s_endpgm
@@ -23,20 +24,26 @@ define amdgpu_kernel void @test_wave32(i32 %arg0, [8 x i32], i32 %saved) {
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX11-NEXT: s_load_b32 s1, s[4:5], 0x24
+; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_cmp_eq_u32 s0, 0
+; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, s1, v0
; GFX11-NEXT: s_cselect_b32 s0, exec_lo, 0
; GFX11-NEXT: s_and_b32 s0, exec_lo, s0
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: s_or_b32 s0, s0, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
; GFX11-NEXT: global_store_b32 v[0:1], v0, off dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_endpgm
entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %mask = icmp ult i32 %tid, %saved
%cond = icmp eq i32 %arg0, 0
- %break = call i32 @llvm.amdgcn.if.break.i32(i1 %cond, i32 %saved)
- store volatile i32 %break, ptr addrspace(1) poison
+ %break = call i1 @llvm.amdgcn.if.break(i1 %cond, i1 %mask)
+ %break.ext = zext i1 %break to i32
+ store volatile i32 %break.ext, ptr addrspace(1) poison
ret void
}
-declare i32 @llvm.amdgcn.if.break.i32(i1, i32)
+declare i32 @llvm.amdgcn.workitem.id.x()
+declare i1 @llvm.amdgcn.if.break(i1, i1)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i64.ll
index 49fe586bd2326..f8b01e6ddd304 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i64.ll
@@ -1,29 +1,34 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -mtriple=amdgcn--amdhsa -mcpu=hawaii < %s | FileCheck -check-prefix=GCN %s
-define amdgpu_kernel void @test_wave64(i32 %arg0, [8 x i32], i64 %saved) {
+define amdgpu_kernel void @test_wave64(i32 %arg0, [8 x i32], i32 %saved) {
; GCN-LABEL: test_wave64:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s2, s[8:9], 0x0
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0xa
+; GCN-NEXT: s_load_dword s0, s[8:9], 0x9
+; GCN-NEXT: s_load_dword s1, s[8:9], 0x0
; GCN-NEXT: s_add_i32 s12, s12, s17
; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_cmp_eq_u32 s2, 0
-; GCN-NEXT: s_cselect_b64 s[2:3], exec, 0
-; GCN-NEXT: s_and_b64 s[2:3], exec, s[2:3]
-; GCN-NEXT: s_or_b64 s[0:1], s[2:3], s[0:1]
-; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_cmp_gt_u32_e32 vcc, s0, v0
+; GCN-NEXT: s_cmp_eq_u32 s1, 0
+; GCN-NEXT: s_cselect_b64 s[0:1], exec, 0
+; GCN-NEXT: s_and_b64 s[0:1], exec, s[0:1]
+; GCN-NEXT: s_or_b64 s[0:1], s[0:1], vcc
+; GCN-NEXT: v_mov_b32_e32 v1, 0
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
; GCN-NEXT: flat_store_dwordx2 v[0:1], v[0:1]
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_endpgm
entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %mask = icmp ult i32 %tid, %saved
%cond = icmp eq i32 %arg0, 0
- %break = call i64 @llvm.amdgcn.if.break.i64(i1 %cond, i64 %saved)
- store volatile i64 %break, ptr addrspace(1) poison
+ %break = call i1 @llvm.amdgcn.if.break(i1 %cond, i1 %mask)
+ %break.ext = zext i1 %break to i64
+ store volatile i64 %break.ext, ptr addrspace(1) poison
ret void
}
-declare i64 @llvm.amdgcn.if.break.i64(i1, i64)
+declare i32 @llvm.amdgcn.workitem.id.x()
+declare i1 @llvm.amdgcn.if.break(i1, i1)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wqm.demote.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wqm.demote.ll
index 44cba2b4904c0..bd759070b5373 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wqm.demote.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wqm.demote.ll
@@ -958,7 +958,7 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
; GFX9-NEXT: s_mov_b64 s[0:1], exec
; GFX9-NEXT: s_wqm_b64 exec, exec
; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
-; GFX9-NEXT: s_mov_b32 s6, 0
+; GFX9-NEXT: s_mov_b32 s8, 0
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
@@ -972,32 +972,38 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
; GFX9-NEXT: .LBB7_3: ; %.continue0.preheader
; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: ; implicit-def: $sgpr4_sgpr5
; GFX9-NEXT: s_branch .LBB7_5
; GFX9-NEXT: .LBB7_4: ; %.continue1
; GFX9-NEXT: ; in Loop: Header=BB7_5 Depth=1
-; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_add_i32 s6, s6, 1
-; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX9-NEXT: s_add_i32 s8, s8, 1
+; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s8, v1
+; GFX9-NEXT: s_or_b64 s[6:7], vcc, s[2:3]
+; GFX9-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_and_b64 s[10:11], exec, s[6:7]
+; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
+; GFX9-NEXT: s_or_b64 s[4:5], s[4:5], s[10:11]
+; GFX9-NEXT: s_or_b64 s[2:3], s[2:3], s[10:11]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX9-NEXT: s_cbranch_execz .LBB7_8
; GFX9-NEXT: .LBB7_5: ; %.continue0
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-NEXT: s_mov_b64 s[8:9], s[0:1]
-; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[8:9]
+; GFX9-NEXT: v_mov_b32_e32 v0, s8
+; GFX9-NEXT: s_mov_b64 s[10:11], s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[10:11]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_mov_b64 s[6:7], exec
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: v_mov_b32_dpp v2, v2 quad_perm:[1,1,1,1] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX9-NEXT: s_nop 1
; GFX9-NEXT: v_subrev_f32_dpp v0, v0, v2 quad_perm:[0,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX9-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec
; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_b64 s[8:9], s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[4:5], s[8:9], s[4:5]
-; GFX9-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
-; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[8:9]
+; GFX9-NEXT: s_and_b64 s[10:11], s[0:1], vcc
+; GFX9-NEXT: s_xor_b64 s[6:7], s[10:11], s[6:7]
+; GFX9-NEXT: s_and_saveexec_b64 s[10:11], s[6:7]
+; GFX9-NEXT: s_xor_b64 s[6:7], exec, s[10:11]
; GFX9-NEXT: s_cbranch_execz .LBB7_4
; GFX9-NEXT: ; %bb.6: ; %.demote1
; GFX9-NEXT: ; in Loop: Header=BB7_5 Depth=1
@@ -1005,11 +1011,11 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
; GFX9-NEXT: s_cbranch_scc0 .LBB7_9
; GFX9-NEXT: ; %bb.7: ; %.demote1
; GFX9-NEXT: ; in Loop: Header=BB7_5 Depth=1
-; GFX9-NEXT: s_wqm_b64 s[8:9], s[0:1]
-; GFX9-NEXT: s_and_b64 exec, exec, s[8:9]
+; GFX9-NEXT: s_wqm_b64 s[10:11], s[0:1]
+; GFX9-NEXT: s_and_b64 exec, exec, s[10:11]
; GFX9-NEXT: s_branch .LBB7_4
; GFX9-NEXT: .LBB7_8: ; %.return
-; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_and_b64 exec, exec, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v0, 0x3c00
; GFX9-NEXT: v_bfrev_b32_e32 v1, 60
@@ -1039,29 +1045,35 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
; GFX10-32-NEXT: .LBB7_3: ; %.continue0.preheader
; GFX10-32-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX10-32-NEXT: s_mov_b32 s2, 0
+; GFX10-32-NEXT: ; implicit-def: $sgpr3
; GFX10-32-NEXT: s_branch .LBB7_5
; GFX10-32-NEXT: .LBB7_4: ; %.continue1
; GFX10-32-NEXT: ; in Loop: Header=BB7_5 Depth=1
-; GFX10-32-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX10-32-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-32-NEXT: s_add_i32 s2, s2, 1
; GFX10-32-NEXT: v_cmp_ge_i32_e32 vcc_lo, s2, v1
-; GFX10-32-NEXT: s_or_b32 s1, vcc_lo, s1
-; GFX10-32-NEXT: s_andn2_b32 exec_lo, exec_lo, s1
+; GFX10-32-NEXT: s_or_b32 s4, vcc_lo, s1
+; GFX10-32-NEXT: s_andn2_b32 s3, s3, exec_lo
+; GFX10-32-NEXT: s_and_b32 s5, exec_lo, s4
+; GFX10-32-NEXT: s_andn2_b32 s1, s1, exec_lo
+; GFX10-32-NEXT: s_or_b32 s3, s3, s5
+; GFX10-32-NEXT: s_or_b32 s1, s1, s5
+; GFX10-32-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-32-NEXT: s_cbranch_execz .LBB7_8
; GFX10-32-NEXT: .LBB7_5: ; %.continue0
; GFX10-32-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-32-NEXT: s_mov_b32 s4, s0
-; GFX10-32-NEXT: s_mov_b32 s3, exec_lo
-; GFX10-32-NEXT: v_cndmask_b32_e64 v0, s2, 0, s4
+; GFX10-32-NEXT: s_mov_b32 s5, s0
+; GFX10-32-NEXT: s_mov_b32 s4, exec_lo
+; GFX10-32-NEXT: v_cndmask_b32_e64 v0, s2, 0, s5
; GFX10-32-NEXT: v_mov_b32_e32 v2, v0
; GFX10-32-NEXT: v_mov_b32_dpp v2, v2 quad_perm:[1,1,1,1] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX10-32-NEXT: v_subrev_f32_dpp v0, v0, v2 quad_perm:[0,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX10-32-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec
; GFX10-32-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v0
-; GFX10-32-NEXT: s_and_b32 s4, s0, vcc_lo
-; GFX10-32-NEXT: s_xor_b32 s3, s4, s3
-; GFX10-32-NEXT: s_and_saveexec_b32 s4, s3
-; GFX10-32-NEXT: s_xor_b32 s3, exec_lo, s4
+; GFX10-32-NEXT: s_and_b32 s5, s0, vcc_lo
+; GFX10-32-NEXT: s_xor_b32 s4, s5, s4
+; GFX10-32-NEXT: s_and_saveexec_b32 s5, s4
+; GFX10-32-NEXT: s_xor_b32 s4, exec_lo, s5
; GFX10-32-NEXT: s_cbranch_execz .LBB7_4
; GFX10-32-NEXT: ; %bb.6: ; %.demote1
; GFX10-32-NEXT: ; in Loop: Header=BB7_5 Depth=1
@@ -1069,11 +1081,11 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
; GFX10-32-NEXT: s_cbranch_scc0 .LBB7_9
; GFX10-32-NEXT: ; %bb.7: ; %.demote1
; GFX10-32-NEXT: ; in Loop: Header=BB7_5 Depth=1
-; GFX10-32-NEXT: s_wqm_b32 s4, s0
-; GFX10-32-NEXT: s_and_b32 exec_lo, exec_lo, s4
+; GFX10-32-NEXT: s_wqm_b32 s5, s0
+; GFX10-32-NEXT: s_and_b32 exec_lo, exec_lo, s5
; GFX10-32-NEXT: s_branch .LBB7_4
; GFX10-32-NEXT: .LBB7_8: ; %.return
-; GFX10-32-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX10-32-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX10-32-NEXT: s_and_b32 exec_lo, exec_lo, s0
; GFX10-32-NEXT: v_mov_b32_e32 v0, 0x3c00
; GFX10-32-NEXT: v_bfrev_b32_e32 v1, 60
@@ -1089,7 +1101,7 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
; GFX10-64-NEXT: s_mov_b64 s[0:1], exec
; GFX10-64-NEXT: s_wqm_b64 exec, exec
; GFX10-64-NEXT: v_cvt_i32_f32_e32 v0, v0
-; GFX10-64-NEXT: s_mov_b32 s6, 0
+; GFX10-64-NEXT: s_mov_b32 s8, 0
; GFX10-64-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; GFX10-64-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX10-64-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
@@ -1103,29 +1115,35 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
; GFX10-64-NEXT: .LBB7_3: ; %.continue0.preheader
; GFX10-64-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX10-64-NEXT: s_mov_b64 s[2:3], 0
+; GFX10-64-NEXT: ; implicit-def: $sgpr4_sgpr5
; GFX10-64-NEXT: s_branch .LBB7_5
; GFX10-64-NEXT: .LBB7_4: ; %.continue1
; GFX10-64-NEXT: ; in Loop: Header=BB7_5 Depth=1
-; GFX10-64-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX10-64-NEXT: s_add_i32 s6, s6, 1
-; GFX10-64-NEXT: v_cmp_ge_i32_e32 vcc, s6, v1
-; GFX10-64-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX10-64-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX10-64-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX10-64-NEXT: s_add_i32 s8, s8, 1
+; GFX10-64-NEXT: v_cmp_ge_i32_e32 vcc, s8, v1
+; GFX10-64-NEXT: s_or_b64 s[6:7], vcc, s[2:3]
+; GFX10-64-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; GFX10-64-NEXT: s_and_b64 s[10:11], exec, s[6:7]
+; GFX10-64-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
+; GFX10-64-NEXT: s_or_b64 s[4:5], s[4:5], s[10:11]
+; GFX10-64-NEXT: s_or_b64 s[2:3], s[2:3], s[10:11]
+; GFX10-64-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX10-64-NEXT: s_cbranch_execz .LBB7_8
; GFX10-64-NEXT: .LBB7_5: ; %.continue0
; GFX10-64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-64-NEXT: s_mov_b64 s[8:9], s[0:1]
-; GFX10-64-NEXT: s_mov_b64 s[4:5], exec
-; GFX10-64-NEXT: v_cndmask_b32_e64 v0, s6, 0, s[8:9]
+; GFX10-64-NEXT: s_mov_b64 s[10:11], s[0:1]
+; GFX10-64-NEXT: s_mov_b64 s[6:7], exec
+; GFX10-64-NEXT: v_cndmask_b32_e64 v0, s8, 0, s[10:11]
; GFX10-64-NEXT: v_mov_b32_e32 v2, v0
; GFX10-64-NEXT: v_mov_b32_dpp v2, v2 quad_perm:[1,1,1,1] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX10-64-NEXT: v_subrev_f32_dpp v0, v0, v2 quad_perm:[0,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX10-64-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec
; GFX10-64-NEXT: v_cmp_eq_f32_e32 vcc, 0, v0
-; GFX10-64-NEXT: s_and_b64 s[8:9], s[0:1], vcc
-; GFX10-64-NEXT: s_xor_b64 s[4:5], s[8:9], s[4:5]
-; GFX10-64-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
-; GFX10-64-NEXT: s_xor_b64 s[4:5], exec, s[8:9]
+; GFX10-64-NEXT: s_and_b64 s[10:11], s[0:1], vcc
+; GFX10-64-NEXT: s_xor_b64 s[6:7], s[10:11], s[6:7]
+; GFX10-64-NEXT: s_and_saveexec_b64 s[10:11], s[6:7]
+; GFX10-64-NEXT: s_xor_b64 s[6:7], exec, s[10:11]
; GFX10-64-NEXT: s_cbranch_execz .LBB7_4
; GFX10-64-NEXT: ; %bb.6: ; %.demote1
; GFX10-64-NEXT: ; in Loop: Header=BB7_5 Depth=1
@@ -1133,11 +1151,11 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
; GFX10-64-NEXT: s_cbranch_scc0 .LBB7_9
; GFX10-64-NEXT: ; %bb.7: ; %.demote1
; GFX10-64-NEXT: ; in Loop: Header=BB7_5 Depth=1
-; GFX10-64-NEXT: s_wqm_b64 s[8:9], s[0:1]
-; GFX10-64-NEXT: s_and_b64 exec, exec, s[8:9]
+; GFX10-64-NEXT: s_wqm_b64 s[10:11], s[0:1]
+; GFX10-64-NEXT: s_and_b64 exec, exec, s[10:11]
; GFX10-64-NEXT: s_branch .LBB7_4
; GFX10-64-NEXT: .LBB7_8: ; %.return
-; GFX10-64-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX10-64-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX10-64-NEXT: s_and_b64 exec, exec, s[0:1]
; GFX10-64-NEXT: v_mov_b32_e32 v0, 0x3c00
; GFX10-64-NEXT: v_bfrev_b32_e32 v1, 60
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-regbanklegalize.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-regbanklegalize.mir
index e3853281c6762..d99c08dd2c60a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-regbanklegalize.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-regbanklegalize.mir
@@ -596,7 +596,7 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY3]]
; CHECK-NEXT: [[COPY4:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[ICMP]](s1)
- ; CHECK-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY4]](s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s1) = SI_IF [[COPY4]](s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: G_BR %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
@@ -606,8 +606,8 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: [[PHI:%[0-9]+]]:vgpr(s32) = G_PHI [[C]](s32), %bb.0, [[C1]](s32), %bb.1
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:sgpr(s32) = COPY [[SI_IF]](s32)
- ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[COPY5]](s32)
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vcc(s1) = COPY [[SI_IF]](s1)
+ ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[COPY5]](s1)
; CHECK-NEXT: G_STORE [[PHI]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
bb.0:
@@ -621,7 +621,7 @@ body: |
%4:sgpr(s32) = G_CONSTANT i32 0
%9:vcc(s1) = G_ICMP intpred(eq), %0(s32), %4
%5:sreg_32_xm0_xexec(s1) = COPY %9(s1)
- %6:sreg_32_xm0_xexec(s32) = SI_IF %5(s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+ %6:sreg_32_xm0_xexec(s1) = SI_IF %5(s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
G_BR %bb.1
bb.1:
@@ -631,8 +631,8 @@ body: |
bb.2:
%8:vgpr(s32) = G_PHI %4(s32), %bb.0, %7(s32), %bb.1
- %10:sgpr(s32) = COPY %6(s32)
- G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %10(s32)
+ %10:vcc(s1) = COPY %6(s1)
+ G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %10(s1)
G_STORE %8(s32), %3(p1) :: (store (s32), addrspace 1)
S_ENDPGM 0
...
@@ -652,30 +652,42 @@ body: |
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -1
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:sreg_32(s1) = G_AMDGPU_COPY_VCC_SCC [[C2]](s32)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x04000000), %bb.1(0x7c000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:sgpr(s32) = G_PHI %7(s32), %bb.1, [[C1]](s32), %bb.0
- ; CHECK-NEXT: [[PHI1:%[0-9]+]]:sgpr(s32) = G_PHI [[C]](s32), %bb.0, %9(s32), %bb.1
- ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[ADD:%[0-9]+]]:sgpr(s32) = G_ADD [[PHI1]], [[C2]]
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](s32)
- ; CHECK-NEXT: [[UITOFP:%[0-9]+]]:vgpr(s32) = G_UITOFP [[COPY3]](s32)
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[AMDGPU_COPY_VCC_SCC]](s1), %bb.0, %9(s1), %bb.1
+ ; CHECK-NEXT: [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %11(s1), %bb.1
+ ; CHECK-NEXT: [[PHI2:%[0-9]+]]:sgpr(s32) = G_PHI [[C]](s32), %bb.0, %13(s32), %bb.1
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:sgpr(s32) = G_ADD [[PHI2]], [[C3]]
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](s32), implicit $exec_lo
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](s32)
+ ; CHECK-NEXT: [[UITOFP:%[0-9]+]]:vgpr(s32) = G_UITOFP [[COPY6]](s32)
; CHECK-NEXT: [[FCMP:%[0-9]+]]:vcc(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[COPY]]
- ; CHECK-NEXT: [[INT:%[0-9]+]]:sgpr(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[FCMP]](s1), [[PHI]](s32)
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:sreg_32_xm0_xexec(s32) = COPY [[INT]](s32)
- ; CHECK-NEXT: SI_LOOP [[COPY4]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vcc(s1) = COPY [[COPY3]](s1)
+ ; CHECK-NEXT: [[INT:%[0-9]+]]:vcc(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[FCMP]](s1), [[COPY7]](s1)
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[INT]](s1)
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[COPY8]](s1)
+ ; CHECK-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY4]](s1), $exec_lo, implicit-def $scc
+ ; CHECK-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY9]](s1), implicit-def $scc
+ ; CHECK-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[COPY8]](s1)
+ ; CHECK-NEXT: SI_LOOP [[COPY8]](s1), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: G_BR %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: [[PHI2:%[0-9]+]]:vgpr(s32) = G_PHI [[ADD]](s32), %bb.1
- ; CHECK-NEXT: [[PHI3:%[0-9]+]]:sgpr(s32) = G_PHI [[INT]](s32), %bb.1
- ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI3]](s32)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[C3]](s32)
- ; CHECK-NEXT: [[MUL:%[0-9]+]]:vgpr(s32) = G_MUL [[PHI2]], [[COPY5]]
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vcc(s1) = COPY [[S_OR_B32_]](s1)
+ ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[COPY11]](s1)
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vgpr(s32) = COPY [[C4]](s32)
+ ; CHECK-NEXT: [[MUL:%[0-9]+]]:vgpr(s32) = G_MUL [[COPY5]], [[COPY12]]
; CHECK-NEXT: G_STORE [[MUL]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
bb.0:
@@ -687,29 +699,40 @@ body: |
%2:vgpr(s32) = COPY $vgpr2
%3:vgpr(p1) = G_MERGE_VALUES %1(s32), %2(s32)
%4:sgpr(s32) = G_CONSTANT i32 -1
- %5:sgpr(s32) = G_CONSTANT i32 0
+ %5:sgpr(s1) = G_CONSTANT i1 false
+ %17:sreg_32(s1) = IMPLICIT_DEF
+ %23:sreg_32(s1) = COPY %5(s1)
bb.1:
successors: %bb.2(0x04000000), %bb.1(0x7c000000)
- %6:sgpr(s32) = G_PHI %17(s32), %bb.1, %5(s32), %bb.0
+ %25:sreg_32(s1) = PHI %23(s1), %bb.0, %24(s1), %bb.1
+ %18:sreg_32(s1) = PHI %17(s1), %bb.0, %16(s1), %bb.1
%8:sgpr(s32) = G_PHI %4(s32), %bb.0, %9(s32), %bb.1
+ %6:sreg_32(s1) = COPY %25(s1)
+ %19:sreg_32(s1) = COPY %18(s1)
%10:sgpr(s32) = G_CONSTANT i32 1
%9:sgpr(s32) = G_ADD %8, %10
+ %15:vgpr(s32) = COPY %9(s32), implicit $exec_lo
%11:sgpr(s32) = G_UITOFP %9(s32)
%12:vcc(s1) = G_FCMP floatpred(ogt), %11(s32), %0
- %17:sgpr(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %12(s1), %6(s32)
- %7:sreg_32_xm0_xexec(s32) = COPY %17(s32)
- SI_LOOP %7(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+ %27:vcc(s1) = COPY %6(s1)
+ %26:vcc(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %12(s1), %27(s1)
+ %7:sreg_32_xm0_xexec(s1) = COPY %26(s1)
+ %20:sreg_32(s1) = COPY %7(s1)
+ %21:sreg_32(s1) = S_ANDN2_B32 %19(s1), $exec_lo, implicit-def $scc
+ %22:sreg_32(s1) = S_AND_B32 $exec_lo, %20(s1), implicit-def $scc
+ %16:sreg_32(s1) = S_OR_B32 %21(s1), %22(s1), implicit-def $scc
+ %24:sreg_32(s1) = COPY %7(s1)
+ SI_LOOP %7(s1), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
G_BR %bb.2
bb.2:
- %13:vgpr(s32) = G_PHI %9(s32), %bb.1
- %14:sgpr(s32) = G_PHI %17(s32), %bb.1
- G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %14(s32)
- %15:sgpr(s32) = G_CONSTANT i32 10
- %16:vgpr(s32) = G_MUL %13, %15
- G_STORE %16(s32), %3(p1) :: (store (s32), addrspace 1)
+ %28:vcc(s1) = COPY %16(s1)
+ G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %28(s1)
+ %13:sgpr(s32) = G_CONSTANT i32 10
+ %14:vgpr(s32) = G_MUL %15, %13
+ G_STORE %14(s32), %3(p1) :: (store (s32), addrspace 1)
S_ENDPGM 0
...
@@ -735,105 +758,123 @@ body: |
; CHECK-NEXT: [[DEF:%[0-9]+]]:sgpr(s32) = G_IMPLICIT_DEF
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
; CHECK-NEXT: [[DEF1:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:sreg_32(s1) = G_AMDGPU_COPY_VCC_SCC [[C]](s32)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.3(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %13(s1), %bb.3
- ; CHECK-NEXT: [[PHI1:%[0-9]+]]:sgpr(s32) = G_PHI %15(s32), %bb.3, [[C]](s32), %bb.0
- ; CHECK-NEXT: [[PHI2:%[0-9]+]]:sgpr(s32) = G_PHI [[C]](s32), %bb.0, %17(s32), %bb.3
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[AMDGPU_COPY_VCC_SCC]](s1), %bb.0, %16(s1), %bb.3
+ ; CHECK-NEXT: [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[DEF2]](s1), %bb.0, %18(s1), %bb.3
+ ; CHECK-NEXT: [[PHI2:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %20(s1), %bb.3
+ ; CHECK-NEXT: [[PHI3:%[0-9]+]]:sgpr(s32) = G_PHI [[C]](s32), %bb.0, %22(s32), %bb.3
; CHECK-NEXT: [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 31
- ; CHECK-NEXT: [[ASHR:%[0-9]+]]:sgpr(s32) = G_ASHR [[PHI2]], [[C1]](i32)
- ; CHECK-NEXT: [[MV3:%[0-9]+]]:sgpr(s64) = G_MERGE_VALUES [[PHI2]](s32), [[ASHR]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:sgpr(s64) = G_SHL [[MV3]], [[C2]](s32)
- ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vgpr(s64) = COPY [[SHL]](s64)
- ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p1) = G_PTR_ADD [[MV1]], [[COPY7]](s64)
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1)
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[PHI2]](s1)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 31
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:sgpr(s32) = G_ASHR [[PHI3]], [[C2]](i32)
+ ; CHECK-NEXT: [[MV3:%[0-9]+]]:sgpr(s64) = G_MERGE_VALUES [[PHI3]](s32), [[ASHR]](s32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:sgpr(s64) = G_SHL [[MV3]], [[C3]](s32)
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vgpr(s64) = COPY [[SHL]](s64)
+ ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p1) = G_PTR_ADD [[MV1]], [[COPY9]](s64)
; CHECK-NEXT: [[LOAD:%[0-9]+]]:vgpr(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C3]](s32)
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[COPY8]]
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[ICMP]](s1)
- ; CHECK-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:sreg_32(s1) = G_AMDGPU_COPY_VCC_SCC [[C4]](s32)
- ; CHECK-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc
- ; CHECK-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[AMDGPU_COPY_VCC_SCC]](s1), implicit-def $scc
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[C4]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[COPY10]]
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[ICMP]](s1)
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC1:%[0-9]+]]:sreg_32(s1) = G_AMDGPU_COPY_VCC_SCC [[C5]](s32)
+ ; CHECK-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc
+ ; CHECK-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[AMDGPU_COPY_VCC_SCC1]](s1), implicit-def $scc
; CHECK-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1)
- ; CHECK-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY9]](s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1)
+ ; CHECK-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s1) = SI_IF [[COPY11]](s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: G_BR %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.5(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[C5:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:sgpr(s64) = G_SHL [[MV3]], [[C5]](s32)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vgpr(s64) = COPY [[SHL1]](s64)
- ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:vgpr(p1) = G_PTR_ADD [[MV2]], [[COPY11]](s64)
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:sgpr(s64) = G_SHL [[MV3]], [[C6]](s32)
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vgpr(s64) = COPY [[SHL1]](s64)
+ ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:vgpr(p1) = G_PTR_ADD [[MV2]], [[COPY13]](s64)
; CHECK-NEXT: [[LOAD1:%[0-9]+]]:vgpr(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s32), addrspace 1)
- ; CHECK-NEXT: [[C6:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vgpr(s32) = COPY [[C6]](s32)
- ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[LOAD1]](s32), [[COPY12]]
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[ICMP1]](s1)
- ; CHECK-NEXT: [[C7:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC1:%[0-9]+]]:sreg_32(s1) = G_AMDGPU_COPY_VCC_SCC [[C7]](s32)
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[AMDGPU_COPY_VCC_SCC1]](s1)
- ; CHECK-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY13]](s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[C7:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vgpr(s32) = COPY [[C7]](s32)
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[LOAD1]](s32), [[COPY14]]
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[ICMP1]](s1)
+ ; CHECK-NEXT: [[C8:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC2:%[0-9]+]]:sreg_32(s1) = G_AMDGPU_COPY_VCC_SCC [[C8]](s32)
+ ; CHECK-NEXT: [[COPY16:%[0-9]+]]:sreg_32(s1) = COPY [[AMDGPU_COPY_VCC_SCC2]](s1)
+ ; CHECK-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = SI_IF [[COPY15]](s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: G_BR %bb.4
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.6(0x04000000), %bb.1(0x7c000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.1, %45(s1), %bb.5
- ; CHECK-NEXT: [[PHI4:%[0-9]+]]:sgpr(s32) = G_PHI %46(s32), %bb.5, [[DEF]](s32), %bb.1
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1)
- ; CHECK-NEXT: [[COPY16:%[0-9]+]]:sgpr(s32) = COPY [[SI_IF]](s32)
- ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[COPY16]](s32)
- ; CHECK-NEXT: [[COPY17:%[0-9]+]]:vcc(s1) = COPY [[COPY15]](s1)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:sgpr(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[COPY17]](s1), [[PHI1]](s32)
- ; CHECK-NEXT: [[COPY18:%[0-9]+]]:sreg_32_xm0_xexec(s32) = COPY [[INT]](s32)
- ; CHECK-NEXT: SI_LOOP [[COPY18]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[PHI4:%[0-9]+]]:sreg_32(s1) = PHI [[PHI]](s1), %bb.1, %53(s1), %bb.5
+ ; CHECK-NEXT: [[PHI5:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.1, %54(s1), %bb.5
+ ; CHECK-NEXT: [[PHI6:%[0-9]+]]:sgpr(s32) = G_PHI %55(s32), %bb.5, [[DEF]](s32), %bb.1
+ ; CHECK-NEXT: [[COPY17:%[0-9]+]]:sreg_32(s1) = COPY [[PHI4]](s1)
+ ; CHECK-NEXT: [[COPY18:%[0-9]+]]:sreg_32(s1) = COPY [[PHI5]](s1)
+ ; CHECK-NEXT: [[COPY19:%[0-9]+]]:vcc(s1) = COPY [[SI_IF]](s1)
+ ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[COPY19]](s1)
+ ; CHECK-NEXT: [[COPY20:%[0-9]+]]:vcc(s1) = COPY [[COPY18]](s1)
+ ; CHECK-NEXT: [[COPY21:%[0-9]+]]:vcc(s1) = COPY [[COPY6]](s1)
+ ; CHECK-NEXT: [[INT:%[0-9]+]]:vcc(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[COPY20]](s1), [[COPY21]](s1)
+ ; CHECK-NEXT: [[COPY22:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[INT]](s1)
+ ; CHECK-NEXT: [[COPY23:%[0-9]+]]:sreg_32(s1) = COPY [[COPY22]](s1)
+ ; CHECK-NEXT: [[COPY24:%[0-9]+]]:sreg_32(s1) = COPY [[COPY22]](s1)
+ ; CHECK-NEXT: [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY7]](s1), $exec_lo, implicit-def $scc
+ ; CHECK-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY24]](s1), implicit-def $scc
+ ; CHECK-NEXT: [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc
+ ; CHECK-NEXT: [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY17]](s1), $exec_lo, implicit-def $scc
+ ; CHECK-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY23]](s1), implicit-def $scc
+ ; CHECK-NEXT: [[S_OR_B32_2:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc
+ ; CHECK-NEXT: SI_LOOP [[COPY22]](s1), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: G_BR %bb.6
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.4:
; CHECK-NEXT: successors: %bb.5(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[C8:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:sgpr(s64) = G_SHL [[MV3]], [[C8]](s32)
- ; CHECK-NEXT: [[COPY19:%[0-9]+]]:vgpr(s64) = COPY [[SHL2]](s64)
- ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:vgpr(p1) = G_PTR_ADD [[MV]], [[COPY19]](s64)
+ ; CHECK-NEXT: [[C9:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:sgpr(s64) = G_SHL [[MV3]], [[C9]](s32)
+ ; CHECK-NEXT: [[COPY25:%[0-9]+]]:vgpr(s64) = COPY [[SHL2]](s64)
+ ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:vgpr(p1) = G_PTR_ADD [[MV]], [[COPY25]](s64)
; CHECK-NEXT: [[LOAD2:%[0-9]+]]:vgpr(s32) = G_LOAD [[PTR_ADD2]](p1) :: (load (s32), addrspace 1)
- ; CHECK-NEXT: [[C9:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[COPY20:%[0-9]+]]:vgpr(s32) = COPY [[C9]](s32)
- ; CHECK-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[LOAD2]], [[COPY20]]
+ ; CHECK-NEXT: [[C10:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[COPY26:%[0-9]+]]:vgpr(s32) = COPY [[C10]](s32)
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[LOAD2]], [[COPY26]]
; CHECK-NEXT: G_STORE [[ADD]](s32), [[PTR_ADD2]](p1) :: (store (s32), addrspace 1)
- ; CHECK-NEXT: [[ADD1:%[0-9]+]]:sgpr(s32) = G_ADD [[PHI2]], [[C9]]
- ; CHECK-NEXT: [[C10:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 100
- ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ult), [[PHI2]](s32), [[C10]]
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP2]], [[C9]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC2:%[0-9]+]]:sreg_32(s1) = G_AMDGPU_COPY_VCC_SCC [[AND]](s32)
- ; CHECK-NEXT: [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY14]](s1), $exec_lo, implicit-def $scc
- ; CHECK-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[AMDGPU_COPY_VCC_SCC2]](s1), implicit-def $scc
- ; CHECK-NEXT: [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc
+ ; CHECK-NEXT: [[ADD1:%[0-9]+]]:sgpr(s32) = G_ADD [[PHI3]], [[C10]]
+ ; CHECK-NEXT: [[C11:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 100
+ ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ult), [[PHI3]](s32), [[C11]]
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP2]], [[C10]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC3:%[0-9]+]]:sreg_32(s1) = G_AMDGPU_COPY_VCC_SCC [[AND]](s32)
+ ; CHECK-NEXT: [[S_ANDN2_B32_3:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY16]](s1), $exec_lo, implicit-def $scc
+ ; CHECK-NEXT: [[S_AND_B32_3:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[AMDGPU_COPY_VCC_SCC3]](s1), implicit-def $scc
+ ; CHECK-NEXT: [[S_OR_B32_3:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_3]](s1), [[S_AND_B32_3]](s1), implicit-def $scc
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.5:
; CHECK-NEXT: successors: %bb.3(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI5:%[0-9]+]]:sreg_32(s1) = PHI [[AMDGPU_COPY_VCC_SCC1]](s1), %bb.2, [[S_OR_B32_1]](s1), %bb.4
- ; CHECK-NEXT: [[PHI6:%[0-9]+]]:sgpr(s32) = G_PHI [[ADD1]](s32), %bb.4, [[DEF]](s32), %bb.2
- ; CHECK-NEXT: [[COPY21:%[0-9]+]]:sreg_32(s1) = COPY [[PHI5]](s1)
- ; CHECK-NEXT: [[COPY22:%[0-9]+]]:sreg_32(s1) = COPY [[COPY21]](s1)
- ; CHECK-NEXT: [[COPY23:%[0-9]+]]:sgpr(s32) = COPY [[SI_IF1]](s32)
- ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[COPY23]](s32)
- ; CHECK-NEXT: [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY10]](s1), $exec_lo, implicit-def $scc
- ; CHECK-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY22]](s1), implicit-def $scc
- ; CHECK-NEXT: [[S_OR_B32_2:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc
+ ; CHECK-NEXT: [[PHI7:%[0-9]+]]:sreg_32(s1) = PHI [[AMDGPU_COPY_VCC_SCC2]](s1), %bb.2, [[S_OR_B32_3]](s1), %bb.4
+ ; CHECK-NEXT: [[PHI8:%[0-9]+]]:sgpr(s32) = G_PHI [[ADD1]](s32), %bb.4, [[DEF]](s32), %bb.2
+ ; CHECK-NEXT: [[COPY27:%[0-9]+]]:sreg_32(s1) = COPY [[PHI7]](s1)
+ ; CHECK-NEXT: [[COPY28:%[0-9]+]]:sreg_32(s1) = COPY [[COPY27]](s1)
+ ; CHECK-NEXT: [[COPY29:%[0-9]+]]:vcc(s1) = COPY [[SI_IF1]](s1)
+ ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[COPY29]](s1)
+ ; CHECK-NEXT: [[S_ANDN2_B32_4:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY12]](s1), $exec_lo, implicit-def $scc
+ ; CHECK-NEXT: [[S_AND_B32_4:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY28]](s1), implicit-def $scc
+ ; CHECK-NEXT: [[S_OR_B32_4:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_4]](s1), [[S_AND_B32_4]](s1), implicit-def $scc
+ ; CHECK-NEXT: [[DEF3:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
; CHECK-NEXT: G_BR %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.6:
- ; CHECK-NEXT: [[PHI7:%[0-9]+]]:sgpr(s32) = G_PHI [[INT]](s32), %bb.3
- ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI7]](s32)
+ ; CHECK-NEXT: [[COPY30:%[0-9]+]]:vcc(s1) = COPY [[S_OR_B32_1]](s1)
+ ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[COPY30]](s1)
; CHECK-NEXT: S_ENDPGM 0
bb.0:
successors: %bb.1(0x80000000)
@@ -850,96 +891,114 @@ body: |
%8:vgpr(p1) = G_MERGE_VALUES %6(s32), %7(s32)
%9:sgpr(s32) = G_IMPLICIT_DEF
%10:sgpr(s32) = G_CONSTANT i32 0
- %11:sreg_32(s1) = IMPLICIT_DEF
+ %11:sgpr(s1) = G_CONSTANT i1 false
+ %12:sreg_32(s1) = IMPLICIT_DEF
+ %66:sreg_32(s1) = IMPLICIT_DEF
+ %73:sreg_32(s1) = COPY %11(s1)
bb.1:
successors: %bb.2(0x40000000), %bb.3(0x40000000)
- %12:sreg_32(s1) = PHI %11(s1), %bb.0, %13(s1), %bb.3
- %14:sgpr(s32) = G_PHI %15(s32), %bb.3, %10(s32), %bb.0
- %16:sgpr(s32) = G_PHI %10(s32), %bb.0, %17(s32), %bb.3
- %18:sreg_32(s1) = COPY %12(s1)
- %19:sgpr(s64) = G_SEXT %16(s32)
- %20:sgpr(s32) = G_CONSTANT i32 2
- %21:sgpr(s64) = G_SHL %19, %20(s32)
- %22:vgpr(p1) = G_PTR_ADD %5, %21(s64)
- %23:vgpr(s32) = G_LOAD %22(p1) :: (load (s32), addrspace 1)
- %24:sgpr(s32) = G_CONSTANT i32 0
- %25:vcc(s1) = G_ICMP intpred(ne), %23(s32), %24
- %26:sreg_32_xm0_xexec(s1) = COPY %25(s1)
+ %76:sreg_32(s1) = PHI %73(s1), %bb.0, %74(s1), %bb.3
+ %67:sreg_32(s1) = PHI %66(s1), %bb.0, %65(s1), %bb.3
+ %13:sreg_32(s1) = PHI %12(s1), %bb.0, %14(s1), %bb.3
+ %17:sgpr(s32) = G_PHI %10(s32), %bb.0, %18(s32), %bb.3
+ %15:sreg_32(s1) = COPY %76(s1)
+ %68:sreg_32(s1) = COPY %67(s1)
+ %19:sreg_32(s1) = COPY %13(s1)
+ %20:sgpr(s64) = G_SEXT %17(s32)
+ %21:sgpr(s32) = G_CONSTANT i32 2
+ %22:sgpr(s64) = G_SHL %20, %21(s32)
+ %23:vgpr(p1) = G_PTR_ADD %5, %22(s64)
+ %24:vgpr(s32) = G_LOAD %23(p1) :: (load (s32), addrspace 1)
+ %25:sgpr(s32) = G_CONSTANT i32 0
+ %81:vcc(s1) = G_ICMP intpred(ne), %24(s32), %25
+ %26:sreg_32_xm0_xexec(s1) = COPY %81(s1)
%27:sgpr(s1) = G_CONSTANT i1 true
%28:sreg_32(s1) = COPY %27(s1)
- %29:sreg_32(s1) = S_ANDN2_B32 %18(s1), $exec_lo, implicit-def $scc
+ %29:sreg_32(s1) = S_ANDN2_B32 %19(s1), $exec_lo, implicit-def $scc
%30:sreg_32(s1) = S_AND_B32 $exec_lo, %28(s1), implicit-def $scc
%31:sreg_32(s1) = S_OR_B32 %29(s1), %30(s1), implicit-def $scc
%32:sreg_32(s1) = COPY %31(s1)
- %33:sreg_32_xm0_xexec(s32) = SI_IF %26(s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+ %33:sreg_32_xm0_xexec(s1) = SI_IF %26(s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
G_BR %bb.2
bb.2:
successors: %bb.4(0x40000000), %bb.5(0x40000000)
%34:sgpr(s32) = G_CONSTANT i32 2
- %35:sgpr(s64) = G_SHL %19, %34(s32)
+ %35:sgpr(s64) = G_SHL %20, %34(s32)
%36:vgpr(p1) = G_PTR_ADD %8, %35(s64)
%37:vgpr(s32) = G_LOAD %36(p1) :: (load (s32), addrspace 1)
%38:sgpr(s32) = G_CONSTANT i32 0
- %39:vcc(s1) = G_ICMP intpred(ne), %37(s32), %38
- %40:sreg_32_xm0_xexec(s1) = COPY %39(s1)
- %41:sgpr(s1) = G_CONSTANT i1 true
+ %82:vcc(s1) = G_ICMP intpred(ne), %37(s32), %38
+ %39:sreg_32_xm0_xexec(s1) = COPY %82(s1)
+ %40:sgpr(s1) = G_CONSTANT i1 true
+ %41:sreg_32(s1) = COPY %40(s1)
%42:sreg_32(s1) = COPY %41(s1)
- %43:sreg_32(s1) = COPY %42(s1)
- %44:sreg_32_xm0_xexec(s32) = SI_IF %40(s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+ %43:sreg_32_xm0_xexec(s1) = SI_IF %39(s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
G_BR %bb.4
bb.3:
successors: %bb.6(0x04000000), %bb.1(0x7c000000)
- %13:sreg_32(s1) = PHI %31(s1), %bb.1, %45(s1), %bb.5
- %17:sgpr(s32) = G_PHI %46(s32), %bb.5, %9(s32), %bb.1
- %47:sreg_32(s1) = COPY %13(s1)
- %48:sgpr(s32) = COPY %33(s32)
- G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %48(s32)
- %49:vcc(s1) = COPY %47(s1)
- %15:sgpr(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %49(s1), %14(s32)
- %50:sreg_32_xm0_xexec(s32) = COPY %15(s32)
- SI_LOOP %50(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+ %75:sreg_32(s1) = PHI %76(s1), %bb.1, %72(s1), %bb.5
+ %14:sreg_32(s1) = PHI %31(s1), %bb.1, %44(s1), %bb.5
+ %18:sgpr(s32) = G_PHI %45(s32), %bb.5, %9(s32), %bb.1
+ %77:sreg_32(s1) = COPY %75(s1)
+ %46:sreg_32(s1) = COPY %14(s1)
+ %83:vcc(s1) = COPY %33(s1)
+ G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %83(s1)
+ %85:vcc(s1) = COPY %46(s1)
+ %86:vcc(s1) = COPY %15(s1)
+ %84:vcc(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %85(s1), %86(s1)
+ %16:sreg_32_xm0_xexec(s1) = COPY %84(s1)
+ %78:sreg_32(s1) = COPY %16(s1)
+ %69:sreg_32(s1) = COPY %16(s1)
+ %70:sreg_32(s1) = S_ANDN2_B32 %68(s1), $exec_lo, implicit-def $scc
+ %71:sreg_32(s1) = S_AND_B32 $exec_lo, %69(s1), implicit-def $scc
+ %65:sreg_32(s1) = S_OR_B32 %70(s1), %71(s1), implicit-def $scc
+ %79:sreg_32(s1) = S_ANDN2_B32 %77(s1), $exec_lo, implicit-def $scc
+ %80:sreg_32(s1) = S_AND_B32 $exec_lo, %78(s1), implicit-def $scc
+ %74:sreg_32(s1) = S_OR_B32 %79(s1), %80(s1), implicit-def $scc
+ SI_LOOP %16(s1), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
G_BR %bb.6
bb.4:
successors: %bb.5(0x80000000)
- %51:sgpr(s32) = G_CONSTANT i32 2
- %52:sgpr(s64) = G_SHL %19, %51(s32)
- %53:vgpr(p1) = G_PTR_ADD %2, %52(s64)
- %54:vgpr(s32) = G_LOAD %53(p1) :: (load (s32), addrspace 1)
- %55:sgpr(s32) = G_CONSTANT i32 1
- %56:vgpr(s32) = G_ADD %54, %55
- G_STORE %56(s32), %53(p1) :: (store (s32), addrspace 1)
- %57:sgpr(s32) = G_ADD %16, %55
- %58:sgpr(s32) = G_CONSTANT i32 100
- %59:sgpr(s1) = G_ICMP intpred(ult), %16(s32), %58
- %60:sreg_32(s1) = COPY %59(s1)
- %61:sreg_32(s1) = S_ANDN2_B32 %43(s1), $exec_lo, implicit-def $scc
- %62:sreg_32(s1) = S_AND_B32 $exec_lo, %60(s1), implicit-def $scc
- %63:sreg_32(s1) = S_OR_B32 %61(s1), %62(s1), implicit-def $scc
+ %47:sgpr(s32) = G_CONSTANT i32 2
+ %48:sgpr(s64) = G_SHL %20, %47(s32)
+ %49:vgpr(p1) = G_PTR_ADD %2, %48(s64)
+ %50:vgpr(s32) = G_LOAD %49(p1) :: (load (s32), addrspace 1)
+ %51:sgpr(s32) = G_CONSTANT i32 1
+ %52:vgpr(s32) = G_ADD %50, %51
+ G_STORE %52(s32), %49(p1) :: (store (s32), addrspace 1)
+ %53:sgpr(s32) = G_ADD %17, %51
+ %54:sgpr(s32) = G_CONSTANT i32 100
+ %55:sgpr(s1) = G_ICMP intpred(ult), %17(s32), %54
+ %56:sreg_32(s1) = COPY %55(s1)
+ %57:sreg_32(s1) = S_ANDN2_B32 %42(s1), $exec_lo, implicit-def $scc
+ %58:sreg_32(s1) = S_AND_B32 $exec_lo, %56(s1), implicit-def $scc
+ %59:sreg_32(s1) = S_OR_B32 %57(s1), %58(s1), implicit-def $scc
bb.5:
successors: %bb.3(0x80000000)
- %64:sreg_32(s1) = PHI %42(s1), %bb.2, %63(s1), %bb.4
- %46:sgpr(s32) = G_PHI %57(s32), %bb.4, %9(s32), %bb.2
- %65:sreg_32(s1) = COPY %64(s1)
- %66:sreg_32(s1) = COPY %65(s1)
- %67:sgpr(s32) = COPY %44(s32)
- G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %67(s32)
- %68:sreg_32(s1) = S_ANDN2_B32 %32(s1), $exec_lo, implicit-def $scc
- %69:sreg_32(s1) = S_AND_B32 $exec_lo, %66(s1), implicit-def $scc
- %45:sreg_32(s1) = S_OR_B32 %68(s1), %69(s1), implicit-def $scc
+ %60:sreg_32(s1) = PHI %41(s1), %bb.2, %59(s1), %bb.4
+ %45:sgpr(s32) = G_PHI %53(s32), %bb.4, %9(s32), %bb.2
+ %61:sreg_32(s1) = COPY %60(s1)
+ %62:sreg_32(s1) = COPY %61(s1)
+ %87:vcc(s1) = COPY %43(s1)
+ G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %87(s1)
+ %63:sreg_32(s1) = S_ANDN2_B32 %32(s1), $exec_lo, implicit-def $scc
+ %64:sreg_32(s1) = S_AND_B32 $exec_lo, %62(s1), implicit-def $scc
+ %44:sreg_32(s1) = S_OR_B32 %63(s1), %64(s1), implicit-def $scc
+ %72:sreg_32(s1) = IMPLICIT_DEF
G_BR %bb.3
bb.6:
- %70:sgpr(s32) = G_PHI %15(s32), %bb.3
- G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %70(s32)
+ %88:vcc(s1) = COPY %65(s1)
+ G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %88(s1)
S_ENDPGM 0
...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.mir
index 091d7617198cf..8c2549f095631 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.mir
@@ -873,7 +873,7 @@ body: |
; OLD_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
; OLD_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(i32) = COPY [[C]](s32)
; OLD_RBS-NEXT: [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY3]]
- ; OLD_RBS-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP]](s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; OLD_RBS-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s1) = SI_IF [[ICMP]](s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
; OLD_RBS-NEXT: G_BR %bb.1
; OLD_RBS-NEXT: {{ $}}
; OLD_RBS-NEXT: bb.1:
@@ -883,9 +883,10 @@ body: |
; OLD_RBS-NEXT: {{ $}}
; OLD_RBS-NEXT: bb.2:
; OLD_RBS-NEXT: [[PHI:%[0-9]+]]:sgpr(s32) = G_PHI [[C]](s32), %bb.0, [[C1]](s32), %bb.1
- ; OLD_RBS-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
- ; OLD_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY [[PHI]](s32)
- ; OLD_RBS-NEXT: G_STORE [[COPY4]](i32), [[MV]](p1) :: (store (s32), addrspace 1)
+ ; OLD_RBS-NEXT: [[COPY4:%[0-9]+]]:sgpr(i1) = COPY [[SI_IF]](s1)
+ ; OLD_RBS-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[COPY4]](i1)
+ ; OLD_RBS-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[PHI]](s32)
+ ; OLD_RBS-NEXT: G_STORE [[COPY5]](i32), [[MV]](p1) :: (store (s32), addrspace 1)
; OLD_RBS-NEXT: S_ENDPGM 0
;
; NEW_RBS-LABEL: name: divergent_phi_with_uniform_inputs
@@ -901,7 +902,7 @@ body: |
; NEW_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
; NEW_RBS-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY3]]
; NEW_RBS-NEXT: [[COPY4:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[ICMP]](s1)
- ; NEW_RBS-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY4]](s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; NEW_RBS-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s1) = SI_IF [[COPY4]](s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
; NEW_RBS-NEXT: G_BR %bb.1
; NEW_RBS-NEXT: {{ $}}
; NEW_RBS-NEXT: bb.1:
@@ -911,8 +912,8 @@ body: |
; NEW_RBS-NEXT: {{ $}}
; NEW_RBS-NEXT: bb.2:
; NEW_RBS-NEXT: [[PHI:%[0-9]+]]:vgpr(s32) = G_PHI [[C]](s32), %bb.0, [[C1]](s32), %bb.1
- ; NEW_RBS-NEXT: [[COPY5:%[0-9]+]]:sgpr(s32) = COPY [[SI_IF]](s32)
- ; NEW_RBS-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[COPY5]](s32)
+ ; NEW_RBS-NEXT: [[COPY5:%[0-9]+]]:vcc(s1) = COPY [[SI_IF]](s1)
+ ; NEW_RBS-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[COPY5]](s1)
; NEW_RBS-NEXT: G_STORE [[PHI]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
; NEW_RBS-NEXT: S_ENDPGM 0
bb.0:
@@ -925,7 +926,7 @@ body: |
%3:_(p1) = G_MERGE_VALUES %1(s32), %2(s32)
%4:_(s32) = G_CONSTANT i32 0
%5:sreg_32_xm0_xexec(s1) = G_ICMP intpred(eq), %0(s32), %4
- %6:sreg_32_xm0_xexec(s32) = SI_IF %5(s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+ %6:sreg_32_xm0_xexec(s1) = SI_IF %5(s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
G_BR %bb.1
bb.1:
@@ -935,7 +936,7 @@ body: |
bb.2:
%8:_(s32) = G_PHI %4(s32), %bb.0, %7(s32), %bb.1
- G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %6(s32)
+ G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %6(s1)
G_STORE %8(s32), %3(p1) :: (store (s32), addrspace 1)
S_ENDPGM 0
...
@@ -956,26 +957,41 @@ body: |
; OLD_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
; OLD_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -1
; OLD_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; OLD_RBS-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[C1]](s32)
+ ; OLD_RBS-NEXT: [[DEF:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
+ ; OLD_RBS-NEXT: [[COPY3:%[0-9]+]]:sreg_32(s1) = COPY [[TRUNC]](s1)
; OLD_RBS-NEXT: {{ $}}
; OLD_RBS-NEXT: bb.1:
; OLD_RBS-NEXT: successors: %bb.2(0x04000000), %bb.1(0x7c000000)
; OLD_RBS-NEXT: {{ $}}
- ; OLD_RBS-NEXT: [[PHI:%[0-9]+]]:sgpr(s32) = G_PHI %7(s32), %bb.1, [[C1]](s32), %bb.0
- ; OLD_RBS-NEXT: [[PHI1:%[0-9]+]]:vgpr(s32) = G_PHI [[C]](s32), %bb.0, %9(s32), %bb.1
+ ; OLD_RBS-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[COPY3]](s1), %bb.0, %9(s1), %bb.1
+ ; OLD_RBS-NEXT: [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %11(s1), %bb.1
+ ; OLD_RBS-NEXT: [[PHI2:%[0-9]+]]:vgpr(s32) = G_PHI [[C]](s32), %bb.0, %13(s32), %bb.1
+ ; OLD_RBS-NEXT: [[COPY4:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
+ ; OLD_RBS-NEXT: [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1)
; OLD_RBS-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; OLD_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(i32) = COPY [[C2]](s32)
- ; OLD_RBS-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[PHI1]], [[COPY3]]
+ ; OLD_RBS-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY [[C2]](s32)
+ ; OLD_RBS-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[PHI2]], [[COPY6]]
+ ; OLD_RBS-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](s32), implicit $exec_lo
; OLD_RBS-NEXT: [[UITOFP:%[0-9]+]]:vgpr(s32) = G_UITOFP [[ADD]](s32)
; OLD_RBS-NEXT: [[FCMP:%[0-9]+]]:vcc(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[COPY]]
- ; OLD_RBS-NEXT: [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[FCMP]](s1), [[PHI]](s32)
- ; OLD_RBS-NEXT: SI_LOOP [[INT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; OLD_RBS-NEXT: [[COPY8:%[0-9]+]]:sgpr(i1) = COPY [[COPY4]](s1)
+ ; OLD_RBS-NEXT: [[INT:%[0-9]+]]:sgpr(i1) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[FCMP]](s1), [[COPY8]](i1)
+ ; OLD_RBS-NEXT: [[COPY9:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[INT]](i1)
+ ; OLD_RBS-NEXT: [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[COPY9]](s1)
+ ; OLD_RBS-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY5]](s1), $exec_lo, implicit-def $scc
+ ; OLD_RBS-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY10]](s1), implicit-def $scc
+ ; OLD_RBS-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
+ ; OLD_RBS-NEXT: [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[COPY9]](s1)
+ ; OLD_RBS-NEXT: SI_LOOP [[COPY9]](s1), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
; OLD_RBS-NEXT: G_BR %bb.2
; OLD_RBS-NEXT: {{ $}}
; OLD_RBS-NEXT: bb.2:
- ; OLD_RBS-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
+ ; OLD_RBS-NEXT: [[COPY12:%[0-9]+]]:sgpr(i1) = COPY [[S_OR_B32_]](s1)
+ ; OLD_RBS-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[COPY12]](i1)
; OLD_RBS-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
- ; OLD_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY [[C3]](s32)
- ; OLD_RBS-NEXT: [[MUL:%[0-9]+]]:vgpr(s32) = G_MUL [[ADD]], [[COPY4]]
+ ; OLD_RBS-NEXT: [[COPY13:%[0-9]+]]:vgpr(i32) = COPY [[C3]](s32)
+ ; OLD_RBS-NEXT: [[MUL:%[0-9]+]]:vgpr(s32) = G_MUL [[COPY7]], [[COPY13]]
; OLD_RBS-NEXT: G_STORE [[MUL]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
; OLD_RBS-NEXT: S_ENDPGM 0
;
@@ -989,29 +1005,42 @@ body: |
; NEW_RBS-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
; NEW_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
; NEW_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -1
- ; NEW_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; NEW_RBS-NEXT: [[DEF:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
+ ; NEW_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+ ; NEW_RBS-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; NEW_RBS-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:sreg_32(s1) = G_AMDGPU_COPY_VCC_SCC [[C2]](s32)
; NEW_RBS-NEXT: {{ $}}
; NEW_RBS-NEXT: bb.1:
; NEW_RBS-NEXT: successors: %bb.2(0x04000000), %bb.1(0x7c000000)
; NEW_RBS-NEXT: {{ $}}
- ; NEW_RBS-NEXT: [[PHI:%[0-9]+]]:sgpr(s32) = G_PHI %15(s32), %bb.1, [[C1]](s32), %bb.0
- ; NEW_RBS-NEXT: [[PHI1:%[0-9]+]]:sgpr(s32) = G_PHI [[C]](s32), %bb.0, %9(s32), %bb.1
- ; NEW_RBS-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; NEW_RBS-NEXT: [[ADD:%[0-9]+]]:sgpr(s32) = G_ADD [[PHI1]], [[C2]]
- ; NEW_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](s32)
- ; NEW_RBS-NEXT: [[UITOFP:%[0-9]+]]:vgpr(s32) = G_UITOFP [[COPY3]](s32)
+ ; NEW_RBS-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[AMDGPU_COPY_VCC_SCC]](s1), %bb.0, %9(s1), %bb.1
+ ; NEW_RBS-NEXT: [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %11(s1), %bb.1
+ ; NEW_RBS-NEXT: [[PHI2:%[0-9]+]]:sgpr(s32) = G_PHI [[C]](s32), %bb.0, %13(s32), %bb.1
+ ; NEW_RBS-NEXT: [[COPY3:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
+ ; NEW_RBS-NEXT: [[COPY4:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1)
+ ; NEW_RBS-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+ ; NEW_RBS-NEXT: [[ADD:%[0-9]+]]:sgpr(s32) = G_ADD [[PHI2]], [[C3]]
+ ; NEW_RBS-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](s32), implicit $exec_lo
+ ; NEW_RBS-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](s32)
+ ; NEW_RBS-NEXT: [[UITOFP:%[0-9]+]]:vgpr(s32) = G_UITOFP [[COPY6]](s32)
; NEW_RBS-NEXT: [[FCMP:%[0-9]+]]:vcc(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[COPY]]
- ; NEW_RBS-NEXT: [[INT:%[0-9]+]]:sgpr(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[FCMP]](s1), [[PHI]](s32)
- ; NEW_RBS-NEXT: [[COPY4:%[0-9]+]]:sreg_32_xm0_xexec(s32) = COPY [[INT]](s32)
- ; NEW_RBS-NEXT: SI_LOOP [[COPY4]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; NEW_RBS-NEXT: [[COPY7:%[0-9]+]]:vcc(s1) = COPY [[COPY3]](s1)
+ ; NEW_RBS-NEXT: [[INT:%[0-9]+]]:vcc(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[FCMP]](s1), [[COPY7]](s1)
+ ; NEW_RBS-NEXT: [[COPY8:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[INT]](s1)
+ ; NEW_RBS-NEXT: [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[COPY8]](s1)
+ ; NEW_RBS-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY4]](s1), $exec_lo, implicit-def $scc
+ ; NEW_RBS-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY9]](s1), implicit-def $scc
+ ; NEW_RBS-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
+ ; NEW_RBS-NEXT: [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[COPY8]](s1)
+ ; NEW_RBS-NEXT: SI_LOOP [[COPY8]](s1), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
; NEW_RBS-NEXT: G_BR %bb.2
; NEW_RBS-NEXT: {{ $}}
; NEW_RBS-NEXT: bb.2:
- ; NEW_RBS-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
- ; NEW_RBS-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
- ; NEW_RBS-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](s32)
- ; NEW_RBS-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[C3]](s32)
- ; NEW_RBS-NEXT: [[MUL:%[0-9]+]]:vgpr(s32) = G_MUL [[COPY5]], [[COPY6]]
+ ; NEW_RBS-NEXT: [[COPY11:%[0-9]+]]:vcc(s1) = COPY [[S_OR_B32_]](s1)
+ ; NEW_RBS-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[COPY11]](s1)
+ ; NEW_RBS-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
+ ; NEW_RBS-NEXT: [[COPY12:%[0-9]+]]:vgpr(s32) = COPY [[C4]](s32)
+ ; NEW_RBS-NEXT: [[MUL:%[0-9]+]]:vgpr(s32) = G_MUL [[COPY5]], [[COPY12]]
; NEW_RBS-NEXT: G_STORE [[MUL]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
; NEW_RBS-NEXT: S_ENDPGM 0
bb.0:
@@ -1023,25 +1052,36 @@ body: |
%2:_(s32) = COPY $vgpr2
%3:_(p1) = G_MERGE_VALUES %1(s32), %2(s32)
%4:_(s32) = G_CONSTANT i32 -1
- %5:_(s32) = G_CONSTANT i32 0
+ %5:_(s1) = G_CONSTANT i1 false
+ %17:sreg_32(s1) = IMPLICIT_DEF
+ %23:sreg_32(s1) = COPY %5(s1)
bb.1:
successors: %bb.2(0x04000000), %bb.1(0x7c000000)
- %6:_(s32) = G_PHI %7(s32), %bb.1, %5(s32), %bb.0
+ %25:sreg_32(s1) = PHI %23(s1), %bb.0, %24(s1), %bb.1
+ %18:sreg_32(s1) = PHI %17(s1), %bb.0, %16(s1), %bb.1
%8:_(s32) = G_PHI %4(s32), %bb.0, %9(s32), %bb.1
+ %6:sreg_32(s1) = COPY %25(s1)
+ %19:sreg_32(s1) = COPY %18(s1)
%10:_(s32) = G_CONSTANT i32 1
%9:_(s32) = G_ADD %8, %10
+ %15:_(s32) = COPY %9(s32), implicit $exec_lo
%11:_(s32) = G_UITOFP %9(s32)
%12:_(s1) = G_FCMP floatpred(ogt), %11(s32), %0
- %7:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %12(s1), %6(s32)
- SI_LOOP %7(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+ %7:sreg_32_xm0_xexec(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %12(s1), %6(s1)
+ %20:sreg_32(s1) = COPY %7(s1)
+ %21:sreg_32(s1) = S_ANDN2_B32 %19(s1), $exec_lo, implicit-def $scc
+ %22:sreg_32(s1) = S_AND_B32 $exec_lo, %20(s1), implicit-def $scc
+ %16:sreg_32(s1) = S_OR_B32 %21(s1), %22(s1), implicit-def $scc
+ %24:sreg_32(s1) = COPY %7(s1)
+ SI_LOOP %7(s1), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
G_BR %bb.2
bb.2:
- G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %7(s32)
+ G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %16(s1)
%13:_(s32) = G_CONSTANT i32 10
- %14:_(s32) = G_MUL %9, %13
+ %14:_(s32) = G_MUL %15, %13
G_STORE %14(s32), %3(p1) :: (store (s32), addrspace 1)
S_ENDPGM 0
...
@@ -1067,103 +1107,126 @@ body: |
; OLD_RBS-NEXT: [[MV2:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
; OLD_RBS-NEXT: [[DEF:%[0-9]+]]:sgpr(s32) = G_IMPLICIT_DEF
; OLD_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; OLD_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; OLD_RBS-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[C1]](s32)
; OLD_RBS-NEXT: [[DEF1:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
+ ; OLD_RBS-NEXT: [[DEF2:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
+ ; OLD_RBS-NEXT: [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[TRUNC]](s1)
; OLD_RBS-NEXT: {{ $}}
; OLD_RBS-NEXT: bb.1:
; OLD_RBS-NEXT: successors: %bb.2(0x40000000), %bb.3(0x40000000)
; OLD_RBS-NEXT: {{ $}}
- ; OLD_RBS-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %13(s1), %bb.3
- ; OLD_RBS-NEXT: [[PHI1:%[0-9]+]]:sgpr(s32) = G_PHI %15(s32), %bb.3, [[C]](s32), %bb.0
- ; OLD_RBS-NEXT: [[PHI2:%[0-9]+]]:vgpr(s32) = G_PHI [[C]](s32), %bb.0, %17(s32), %bb.3
- ; OLD_RBS-NEXT: [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
- ; OLD_RBS-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[PHI2]](s32)
- ; OLD_RBS-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 31
- ; OLD_RBS-NEXT: [[ASHR:%[0-9]+]]:vgpr(i32) = G_ASHR [[COPY7]], [[C1]](s32)
- ; OLD_RBS-NEXT: [[MV3:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY7]](i32), [[ASHR]](i32)
- ; OLD_RBS-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
- ; OLD_RBS-NEXT: [[COPY8:%[0-9]+]]:vgpr(i32) = COPY [[C2]](s32)
- ; OLD_RBS-NEXT: [[SHL:%[0-9]+]]:vgpr(s64) = G_SHL [[MV3]], [[COPY8]](i32)
+ ; OLD_RBS-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[COPY6]](s1), %bb.0, %16(s1), %bb.3
+ ; OLD_RBS-NEXT: [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[DEF2]](s1), %bb.0, %18(s1), %bb.3
+ ; OLD_RBS-NEXT: [[PHI2:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %20(s1), %bb.3
+ ; OLD_RBS-NEXT: [[PHI3:%[0-9]+]]:vgpr(s32) = G_PHI [[C]](s32), %bb.0, %22(s32), %bb.3
+ ; OLD_RBS-NEXT: [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
+ ; OLD_RBS-NEXT: [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1)
+ ; OLD_RBS-NEXT: [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[PHI2]](s1)
+ ; OLD_RBS-NEXT: [[COPY10:%[0-9]+]]:vgpr(i32) = COPY [[PHI3]](s32)
+ ; OLD_RBS-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 31
+ ; OLD_RBS-NEXT: [[ASHR:%[0-9]+]]:vgpr(i32) = G_ASHR [[COPY10]], [[C2]](s32)
+ ; OLD_RBS-NEXT: [[MV3:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY10]](i32), [[ASHR]](i32)
+ ; OLD_RBS-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
+ ; OLD_RBS-NEXT: [[COPY11:%[0-9]+]]:vgpr(i32) = COPY [[C3]](s32)
+ ; OLD_RBS-NEXT: [[SHL:%[0-9]+]]:vgpr(s64) = G_SHL [[MV3]], [[COPY11]](i32)
; OLD_RBS-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p1) = G_PTR_ADD [[MV1]], [[SHL]](s64)
; OLD_RBS-NEXT: [[LOAD:%[0-9]+]]:vgpr(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1)
- ; OLD_RBS-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; OLD_RBS-NEXT: [[COPY9:%[0-9]+]]:vgpr(i32) = COPY [[C3]](s32)
- ; OLD_RBS-NEXT: [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[COPY9]]
- ; OLD_RBS-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; OLD_RBS-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[C4]](s32)
- ; OLD_RBS-NEXT: [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[TRUNC]](s1)
- ; OLD_RBS-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc
- ; OLD_RBS-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY10]](s1), implicit-def $scc
+ ; OLD_RBS-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; OLD_RBS-NEXT: [[COPY12:%[0-9]+]]:vgpr(i32) = COPY [[C4]](s32)
+ ; OLD_RBS-NEXT: [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[COPY12]]
+ ; OLD_RBS-NEXT: [[C5:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+ ; OLD_RBS-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s1) = G_TRUNC [[C5]](s32)
+ ; OLD_RBS-NEXT: [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[TRUNC1]](s1)
+ ; OLD_RBS-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY9]](s1), $exec_lo, implicit-def $scc
+ ; OLD_RBS-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY13]](s1), implicit-def $scc
; OLD_RBS-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
- ; OLD_RBS-NEXT: [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1)
- ; OLD_RBS-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP]](s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; OLD_RBS-NEXT: [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1)
+ ; OLD_RBS-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s1) = SI_IF [[ICMP]](s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
; OLD_RBS-NEXT: G_BR %bb.2
; OLD_RBS-NEXT: {{ $}}
; OLD_RBS-NEXT: bb.2:
; OLD_RBS-NEXT: successors: %bb.4(0x40000000), %bb.5(0x40000000)
; OLD_RBS-NEXT: {{ $}}
- ; OLD_RBS-NEXT: [[C5:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
- ; OLD_RBS-NEXT: [[COPY12:%[0-9]+]]:vgpr(i32) = COPY [[C5]](s32)
- ; OLD_RBS-NEXT: [[SHL1:%[0-9]+]]:vgpr(s64) = G_SHL [[MV3]], [[COPY12]](i32)
+ ; OLD_RBS-NEXT: [[C6:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
+ ; OLD_RBS-NEXT: [[COPY15:%[0-9]+]]:vgpr(i32) = COPY [[C6]](s32)
+ ; OLD_RBS-NEXT: [[SHL1:%[0-9]+]]:vgpr(s64) = G_SHL [[MV3]], [[COPY15]](i32)
; OLD_RBS-NEXT: [[PTR_ADD1:%[0-9]+]]:vgpr(p1) = G_PTR_ADD [[MV2]], [[SHL1]](s64)
; OLD_RBS-NEXT: [[LOAD1:%[0-9]+]]:vgpr(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s32), addrspace 1)
- ; OLD_RBS-NEXT: [[C6:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; OLD_RBS-NEXT: [[COPY13:%[0-9]+]]:vgpr(i32) = COPY [[C6]](s32)
- ; OLD_RBS-NEXT: [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD1]](s32), [[COPY13]]
- ; OLD_RBS-NEXT: [[C7:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; OLD_RBS-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s1) = G_TRUNC [[C7]](s32)
- ; OLD_RBS-NEXT: [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[TRUNC1]](s1)
- ; OLD_RBS-NEXT: [[COPY15:%[0-9]+]]:sreg_32(s1) = COPY [[COPY14]](s1)
- ; OLD_RBS-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP1]](s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; OLD_RBS-NEXT: [[C7:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; OLD_RBS-NEXT: [[COPY16:%[0-9]+]]:vgpr(i32) = COPY [[C7]](s32)
+ ; OLD_RBS-NEXT: [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD1]](s32), [[COPY16]]
+ ; OLD_RBS-NEXT: [[C8:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+ ; OLD_RBS-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s1) = G_TRUNC [[C8]](s32)
+ ; OLD_RBS-NEXT: [[COPY17:%[0-9]+]]:sreg_32(s1) = COPY [[TRUNC2]](s1)
+ ; OLD_RBS-NEXT: [[COPY18:%[0-9]+]]:sreg_32(s1) = COPY [[COPY17]](s1)
+ ; OLD_RBS-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = SI_IF [[ICMP1]](s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
; OLD_RBS-NEXT: G_BR %bb.4
; OLD_RBS-NEXT: {{ $}}
; OLD_RBS-NEXT: bb.3:
; OLD_RBS-NEXT: successors: %bb.6(0x04000000), %bb.1(0x7c000000)
; OLD_RBS-NEXT: {{ $}}
- ; OLD_RBS-NEXT: [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.1, %43(s1), %bb.5
- ; OLD_RBS-NEXT: [[PHI4:%[0-9]+]]:vgpr(s32) = G_PHI %44(s32), %bb.5, [[DEF]](s32), %bb.1
- ; OLD_RBS-NEXT: [[COPY16:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1)
- ; OLD_RBS-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
- ; OLD_RBS-NEXT: [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[COPY16]](s1), [[PHI1]](s32)
- ; OLD_RBS-NEXT: SI_LOOP [[INT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; OLD_RBS-NEXT: [[PHI4:%[0-9]+]]:sreg_32(s1) = PHI [[PHI]](s1), %bb.1, %51(s1), %bb.5
+ ; OLD_RBS-NEXT: [[PHI5:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.1, %52(s1), %bb.5
+ ; OLD_RBS-NEXT: [[PHI6:%[0-9]+]]:vgpr(s32) = G_PHI %53(s32), %bb.5, [[DEF]](s32), %bb.1
+ ; OLD_RBS-NEXT: [[COPY19:%[0-9]+]]:sreg_32(s1) = COPY [[PHI4]](s1)
+ ; OLD_RBS-NEXT: [[COPY20:%[0-9]+]]:sreg_32(s1) = COPY [[PHI5]](s1)
+ ; OLD_RBS-NEXT: [[COPY21:%[0-9]+]]:sgpr(i1) = COPY [[SI_IF]](s1)
+ ; OLD_RBS-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[COPY21]](i1)
+ ; OLD_RBS-NEXT: [[COPY22:%[0-9]+]]:sgpr(i1) = COPY [[COPY7]](s1)
+ ; OLD_RBS-NEXT: [[INT:%[0-9]+]]:sgpr(i1) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[COPY20]](s1), [[COPY22]](i1)
+ ; OLD_RBS-NEXT: [[COPY23:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[INT]](i1)
+ ; OLD_RBS-NEXT: [[COPY24:%[0-9]+]]:sreg_32(s1) = COPY [[COPY23]](s1)
+ ; OLD_RBS-NEXT: [[COPY25:%[0-9]+]]:sreg_32(s1) = COPY [[COPY23]](s1)
+ ; OLD_RBS-NEXT: [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc
+ ; OLD_RBS-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY25]](s1), implicit-def $scc
+ ; OLD_RBS-NEXT: [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc
+ ; OLD_RBS-NEXT: [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY19]](s1), $exec_lo, implicit-def $scc
+ ; OLD_RBS-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY24]](s1), implicit-def $scc
+ ; OLD_RBS-NEXT: [[S_OR_B32_2:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc
+ ; OLD_RBS-NEXT: SI_LOOP [[COPY23]](s1), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
; OLD_RBS-NEXT: G_BR %bb.6
; OLD_RBS-NEXT: {{ $}}
; OLD_RBS-NEXT: bb.4:
; OLD_RBS-NEXT: successors: %bb.5(0x80000000)
; OLD_RBS-NEXT: {{ $}}
- ; OLD_RBS-NEXT: [[C8:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
- ; OLD_RBS-NEXT: [[COPY17:%[0-9]+]]:vgpr(i32) = COPY [[C8]](s32)
- ; OLD_RBS-NEXT: [[SHL2:%[0-9]+]]:vgpr(s64) = G_SHL [[MV3]], [[COPY17]](i32)
+ ; OLD_RBS-NEXT: [[C9:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
+ ; OLD_RBS-NEXT: [[COPY26:%[0-9]+]]:vgpr(i32) = COPY [[C9]](s32)
+ ; OLD_RBS-NEXT: [[SHL2:%[0-9]+]]:vgpr(s64) = G_SHL [[MV3]], [[COPY26]](i32)
; OLD_RBS-NEXT: [[PTR_ADD2:%[0-9]+]]:vgpr(p1) = G_PTR_ADD [[MV]], [[SHL2]](s64)
; OLD_RBS-NEXT: [[LOAD2:%[0-9]+]]:vgpr(s32) = G_LOAD [[PTR_ADD2]](p1) :: (load (s32), addrspace 1)
- ; OLD_RBS-NEXT: [[C9:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; OLD_RBS-NEXT: [[COPY18:%[0-9]+]]:vgpr(i32) = COPY [[C9]](s32)
- ; OLD_RBS-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[LOAD2]], [[COPY18]]
+ ; OLD_RBS-NEXT: [[C10:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+ ; OLD_RBS-NEXT: [[COPY27:%[0-9]+]]:vgpr(i32) = COPY [[C10]](s32)
+ ; OLD_RBS-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[LOAD2]], [[COPY27]]
; OLD_RBS-NEXT: G_STORE [[ADD]](s32), [[PTR_ADD2]](p1) :: (store (s32), addrspace 1)
- ; OLD_RBS-NEXT: [[COPY19:%[0-9]+]]:vgpr(i32) = COPY [[C9]](s32)
- ; OLD_RBS-NEXT: [[ADD1:%[0-9]+]]:vgpr(s32) = G_ADD [[PHI2]], [[COPY19]]
- ; OLD_RBS-NEXT: [[C10:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 100
- ; OLD_RBS-NEXT: [[COPY20:%[0-9]+]]:vgpr(i32) = COPY [[C10]](s32)
- ; OLD_RBS-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ult), [[PHI2]](s32), [[COPY20]]
- ; OLD_RBS-NEXT: [[COPY21:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1)
- ; OLD_RBS-NEXT: [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY15]](s1), $exec_lo, implicit-def $scc
- ; OLD_RBS-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY21]](s1), implicit-def $scc
- ; OLD_RBS-NEXT: [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc
+ ; OLD_RBS-NEXT: [[COPY28:%[0-9]+]]:vgpr(i32) = COPY [[C10]](s32)
+ ; OLD_RBS-NEXT: [[ADD1:%[0-9]+]]:vgpr(s32) = G_ADD [[PHI3]], [[COPY28]]
+ ; OLD_RBS-NEXT: [[C11:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 100
+ ; OLD_RBS-NEXT: [[COPY29:%[0-9]+]]:vgpr(i32) = COPY [[C11]](s32)
+ ; OLD_RBS-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ult), [[PHI3]](s32), [[COPY29]]
+ ; OLD_RBS-NEXT: [[COPY30:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1)
+ ; OLD_RBS-NEXT: [[S_ANDN2_B32_3:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY18]](s1), $exec_lo, implicit-def $scc
+ ; OLD_RBS-NEXT: [[S_AND_B32_3:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY30]](s1), implicit-def $scc
+ ; OLD_RBS-NEXT: [[S_OR_B32_3:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_3]](s1), [[S_AND_B32_3]](s1), implicit-def $scc
; OLD_RBS-NEXT: {{ $}}
; OLD_RBS-NEXT: bb.5:
; OLD_RBS-NEXT: successors: %bb.3(0x80000000)
; OLD_RBS-NEXT: {{ $}}
- ; OLD_RBS-NEXT: [[PHI5:%[0-9]+]]:sreg_32(s1) = PHI [[COPY14]](s1), %bb.2, [[S_OR_B32_1]](s1), %bb.4
- ; OLD_RBS-NEXT: [[PHI6:%[0-9]+]]:vgpr(s32) = G_PHI [[ADD1]](s32), %bb.4, [[DEF]](s32), %bb.2
- ; OLD_RBS-NEXT: [[COPY22:%[0-9]+]]:sreg_32(s1) = COPY [[PHI5]](s1)
- ; OLD_RBS-NEXT: [[COPY23:%[0-9]+]]:sreg_32(s1) = COPY [[COPY22]](s1)
- ; OLD_RBS-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF1]](s32)
- ; OLD_RBS-NEXT: [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY11]](s1), $exec_lo, implicit-def $scc
- ; OLD_RBS-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY23]](s1), implicit-def $scc
- ; OLD_RBS-NEXT: [[S_OR_B32_2:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc
+ ; OLD_RBS-NEXT: [[PHI7:%[0-9]+]]:sreg_32(s1) = PHI [[COPY17]](s1), %bb.2, [[S_OR_B32_3]](s1), %bb.4
+ ; OLD_RBS-NEXT: [[PHI8:%[0-9]+]]:vgpr(s32) = G_PHI [[ADD1]](s32), %bb.4, [[DEF]](s32), %bb.2
+ ; OLD_RBS-NEXT: [[COPY31:%[0-9]+]]:sreg_32(s1) = COPY [[PHI7]](s1)
+ ; OLD_RBS-NEXT: [[COPY32:%[0-9]+]]:sreg_32(s1) = COPY [[COPY31]](s1)
+ ; OLD_RBS-NEXT: [[COPY33:%[0-9]+]]:sgpr(i1) = COPY [[SI_IF1]](s1)
+ ; OLD_RBS-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[COPY33]](i1)
+ ; OLD_RBS-NEXT: [[S_ANDN2_B32_4:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY14]](s1), $exec_lo, implicit-def $scc
+ ; OLD_RBS-NEXT: [[S_AND_B32_4:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY32]](s1), implicit-def $scc
+ ; OLD_RBS-NEXT: [[S_OR_B32_4:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_4]](s1), [[S_AND_B32_4]](s1), implicit-def $scc
+ ; OLD_RBS-NEXT: [[DEF3:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
; OLD_RBS-NEXT: G_BR %bb.3
; OLD_RBS-NEXT: {{ $}}
; OLD_RBS-NEXT: bb.6:
- ; OLD_RBS-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
+ ; OLD_RBS-NEXT: [[COPY34:%[0-9]+]]:sgpr(i1) = COPY [[S_OR_B32_1]](s1)
+ ; OLD_RBS-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[COPY34]](i1)
; OLD_RBS-NEXT: S_ENDPGM 0
;
; NEW_RBS-LABEL: name: loop_with_2breaks
@@ -1183,104 +1246,123 @@ body: |
; NEW_RBS-NEXT: [[DEF:%[0-9]+]]:sgpr(s32) = G_IMPLICIT_DEF
; NEW_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
; NEW_RBS-NEXT: [[DEF1:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
+ ; NEW_RBS-NEXT: [[DEF2:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
+ ; NEW_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+ ; NEW_RBS-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:sreg_32(s1) = G_AMDGPU_COPY_VCC_SCC [[C]](s32)
; NEW_RBS-NEXT: {{ $}}
; NEW_RBS-NEXT: bb.1:
; NEW_RBS-NEXT: successors: %bb.2(0x40000000), %bb.3(0x40000000)
; NEW_RBS-NEXT: {{ $}}
- ; NEW_RBS-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %13(s1), %bb.3
- ; NEW_RBS-NEXT: [[PHI1:%[0-9]+]]:sgpr(s32) = G_PHI %67(s32), %bb.3, [[C]](s32), %bb.0
- ; NEW_RBS-NEXT: [[PHI2:%[0-9]+]]:sgpr(s32) = G_PHI [[C]](s32), %bb.0, %17(s32), %bb.3
+ ; NEW_RBS-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[AMDGPU_COPY_VCC_SCC]](s1), %bb.0, %16(s1), %bb.3
+ ; NEW_RBS-NEXT: [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[DEF2]](s1), %bb.0, %18(s1), %bb.3
+ ; NEW_RBS-NEXT: [[PHI2:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %20(s1), %bb.3
+ ; NEW_RBS-NEXT: [[PHI3:%[0-9]+]]:sgpr(s32) = G_PHI [[C]](s32), %bb.0, %22(s32), %bb.3
; NEW_RBS-NEXT: [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
- ; NEW_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 31
- ; NEW_RBS-NEXT: [[ASHR:%[0-9]+]]:sgpr(s32) = G_ASHR [[PHI2]], [[C1]](i32)
- ; NEW_RBS-NEXT: [[MV3:%[0-9]+]]:sgpr(s64) = G_MERGE_VALUES [[PHI2]](s32), [[ASHR]](s32)
- ; NEW_RBS-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
- ; NEW_RBS-NEXT: [[SHL:%[0-9]+]]:sgpr(s64) = G_SHL [[MV3]], [[C2]](s32)
- ; NEW_RBS-NEXT: [[COPY7:%[0-9]+]]:vgpr(s64) = COPY [[SHL]](s64)
- ; NEW_RBS-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p1) = G_PTR_ADD [[MV1]], [[COPY7]](s64)
+ ; NEW_RBS-NEXT: [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1)
+ ; NEW_RBS-NEXT: [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[PHI2]](s1)
+ ; NEW_RBS-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 31
+ ; NEW_RBS-NEXT: [[ASHR:%[0-9]+]]:sgpr(s32) = G_ASHR [[PHI3]], [[C2]](i32)
+ ; NEW_RBS-NEXT: [[MV3:%[0-9]+]]:sgpr(s64) = G_MERGE_VALUES [[PHI3]](s32), [[ASHR]](s32)
+ ; NEW_RBS-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
+ ; NEW_RBS-NEXT: [[SHL:%[0-9]+]]:sgpr(s64) = G_SHL [[MV3]], [[C3]](s32)
+ ; NEW_RBS-NEXT: [[COPY9:%[0-9]+]]:vgpr(s64) = COPY [[SHL]](s64)
+ ; NEW_RBS-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p1) = G_PTR_ADD [[MV1]], [[COPY9]](s64)
; NEW_RBS-NEXT: [[LOAD:%[0-9]+]]:vgpr(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1)
- ; NEW_RBS-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; NEW_RBS-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C3]](s32)
- ; NEW_RBS-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[COPY8]]
- ; NEW_RBS-NEXT: [[COPY9:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[ICMP]](s1)
- ; NEW_RBS-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; NEW_RBS-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:sreg_32(s1) = G_AMDGPU_COPY_VCC_SCC [[C4]](s32)
- ; NEW_RBS-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc
- ; NEW_RBS-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[AMDGPU_COPY_VCC_SCC]](s1), implicit-def $scc
+ ; NEW_RBS-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; NEW_RBS-NEXT: [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[C4]](s32)
+ ; NEW_RBS-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[COPY10]]
+ ; NEW_RBS-NEXT: [[COPY11:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[ICMP]](s1)
+ ; NEW_RBS-NEXT: [[C5:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+ ; NEW_RBS-NEXT: [[AMDGPU_COPY_VCC_SCC1:%[0-9]+]]:sreg_32(s1) = G_AMDGPU_COPY_VCC_SCC [[C5]](s32)
+ ; NEW_RBS-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc
+ ; NEW_RBS-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[AMDGPU_COPY_VCC_SCC1]](s1), implicit-def $scc
; NEW_RBS-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
- ; NEW_RBS-NEXT: [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1)
- ; NEW_RBS-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY9]](s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; NEW_RBS-NEXT: [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1)
+ ; NEW_RBS-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s1) = SI_IF [[COPY11]](s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
; NEW_RBS-NEXT: G_BR %bb.2
; NEW_RBS-NEXT: {{ $}}
; NEW_RBS-NEXT: bb.2:
; NEW_RBS-NEXT: successors: %bb.4(0x40000000), %bb.5(0x40000000)
; NEW_RBS-NEXT: {{ $}}
- ; NEW_RBS-NEXT: [[C5:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
- ; NEW_RBS-NEXT: [[SHL1:%[0-9]+]]:sgpr(s64) = G_SHL [[MV3]], [[C5]](s32)
- ; NEW_RBS-NEXT: [[COPY11:%[0-9]+]]:vgpr(s64) = COPY [[SHL1]](s64)
- ; NEW_RBS-NEXT: [[PTR_ADD1:%[0-9]+]]:vgpr(p1) = G_PTR_ADD [[MV2]], [[COPY11]](s64)
+ ; NEW_RBS-NEXT: [[C6:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
+ ; NEW_RBS-NEXT: [[SHL1:%[0-9]+]]:sgpr(s64) = G_SHL [[MV3]], [[C6]](s32)
+ ; NEW_RBS-NEXT: [[COPY13:%[0-9]+]]:vgpr(s64) = COPY [[SHL1]](s64)
+ ; NEW_RBS-NEXT: [[PTR_ADD1:%[0-9]+]]:vgpr(p1) = G_PTR_ADD [[MV2]], [[COPY13]](s64)
; NEW_RBS-NEXT: [[LOAD1:%[0-9]+]]:vgpr(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s32), addrspace 1)
- ; NEW_RBS-NEXT: [[C6:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; NEW_RBS-NEXT: [[COPY12:%[0-9]+]]:vgpr(s32) = COPY [[C6]](s32)
- ; NEW_RBS-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[LOAD1]](s32), [[COPY12]]
- ; NEW_RBS-NEXT: [[COPY13:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[ICMP1]](s1)
- ; NEW_RBS-NEXT: [[C7:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; NEW_RBS-NEXT: [[AMDGPU_COPY_VCC_SCC1:%[0-9]+]]:sreg_32(s1) = G_AMDGPU_COPY_VCC_SCC [[C7]](s32)
- ; NEW_RBS-NEXT: [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[AMDGPU_COPY_VCC_SCC1]](s1)
- ; NEW_RBS-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY13]](s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; NEW_RBS-NEXT: [[C7:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; NEW_RBS-NEXT: [[COPY14:%[0-9]+]]:vgpr(s32) = COPY [[C7]](s32)
+ ; NEW_RBS-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[LOAD1]](s32), [[COPY14]]
+ ; NEW_RBS-NEXT: [[COPY15:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[ICMP1]](s1)
+ ; NEW_RBS-NEXT: [[C8:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+ ; NEW_RBS-NEXT: [[AMDGPU_COPY_VCC_SCC2:%[0-9]+]]:sreg_32(s1) = G_AMDGPU_COPY_VCC_SCC [[C8]](s32)
+ ; NEW_RBS-NEXT: [[COPY16:%[0-9]+]]:sreg_32(s1) = COPY [[AMDGPU_COPY_VCC_SCC2]](s1)
+ ; NEW_RBS-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = SI_IF [[COPY15]](s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
; NEW_RBS-NEXT: G_BR %bb.4
; NEW_RBS-NEXT: {{ $}}
; NEW_RBS-NEXT: bb.3:
; NEW_RBS-NEXT: successors: %bb.6(0x04000000), %bb.1(0x7c000000)
; NEW_RBS-NEXT: {{ $}}
- ; NEW_RBS-NEXT: [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.1, %43(s1), %bb.5
- ; NEW_RBS-NEXT: [[PHI4:%[0-9]+]]:sgpr(s32) = G_PHI %44(s32), %bb.5, [[DEF]](s32), %bb.1
- ; NEW_RBS-NEXT: [[COPY15:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1)
- ; NEW_RBS-NEXT: [[COPY16:%[0-9]+]]:sgpr(s32) = COPY [[SI_IF]](s32)
- ; NEW_RBS-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[COPY16]](s32)
- ; NEW_RBS-NEXT: [[COPY17:%[0-9]+]]:vcc(s1) = COPY [[COPY15]](s1)
- ; NEW_RBS-NEXT: [[INT:%[0-9]+]]:sgpr(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[COPY17]](s1), [[PHI1]](s32)
- ; NEW_RBS-NEXT: [[COPY18:%[0-9]+]]:sreg_32_xm0_xexec(s32) = COPY [[INT]](s32)
- ; NEW_RBS-NEXT: SI_LOOP [[COPY18]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; NEW_RBS-NEXT: [[PHI4:%[0-9]+]]:sreg_32(s1) = PHI [[PHI]](s1), %bb.1, %51(s1), %bb.5
+ ; NEW_RBS-NEXT: [[PHI5:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.1, %52(s1), %bb.5
+ ; NEW_RBS-NEXT: [[PHI6:%[0-9]+]]:sgpr(s32) = G_PHI %53(s32), %bb.5, [[DEF]](s32), %bb.1
+ ; NEW_RBS-NEXT: [[COPY17:%[0-9]+]]:sreg_32(s1) = COPY [[PHI4]](s1)
+ ; NEW_RBS-NEXT: [[COPY18:%[0-9]+]]:sreg_32(s1) = COPY [[PHI5]](s1)
+ ; NEW_RBS-NEXT: [[COPY19:%[0-9]+]]:vcc(s1) = COPY [[SI_IF]](s1)
+ ; NEW_RBS-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[COPY19]](s1)
+ ; NEW_RBS-NEXT: [[COPY20:%[0-9]+]]:vcc(s1) = COPY [[COPY18]](s1)
+ ; NEW_RBS-NEXT: [[COPY21:%[0-9]+]]:vcc(s1) = COPY [[COPY6]](s1)
+ ; NEW_RBS-NEXT: [[INT:%[0-9]+]]:vcc(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[COPY20]](s1), [[COPY21]](s1)
+ ; NEW_RBS-NEXT: [[COPY22:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[INT]](s1)
+ ; NEW_RBS-NEXT: [[COPY23:%[0-9]+]]:sreg_32(s1) = COPY [[COPY22]](s1)
+ ; NEW_RBS-NEXT: [[COPY24:%[0-9]+]]:sreg_32(s1) = COPY [[COPY22]](s1)
+ ; NEW_RBS-NEXT: [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY7]](s1), $exec_lo, implicit-def $scc
+ ; NEW_RBS-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY24]](s1), implicit-def $scc
+ ; NEW_RBS-NEXT: [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc
+ ; NEW_RBS-NEXT: [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY17]](s1), $exec_lo, implicit-def $scc
+ ; NEW_RBS-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY23]](s1), implicit-def $scc
+ ; NEW_RBS-NEXT: [[S_OR_B32_2:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc
+ ; NEW_RBS-NEXT: SI_LOOP [[COPY22]](s1), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
; NEW_RBS-NEXT: G_BR %bb.6
; NEW_RBS-NEXT: {{ $}}
; NEW_RBS-NEXT: bb.4:
; NEW_RBS-NEXT: successors: %bb.5(0x80000000)
; NEW_RBS-NEXT: {{ $}}
- ; NEW_RBS-NEXT: [[C8:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
- ; NEW_RBS-NEXT: [[SHL2:%[0-9]+]]:sgpr(s64) = G_SHL [[MV3]], [[C8]](s32)
- ; NEW_RBS-NEXT: [[COPY19:%[0-9]+]]:vgpr(s64) = COPY [[SHL2]](s64)
- ; NEW_RBS-NEXT: [[PTR_ADD2:%[0-9]+]]:vgpr(p1) = G_PTR_ADD [[MV]], [[COPY19]](s64)
+ ; NEW_RBS-NEXT: [[C9:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
+ ; NEW_RBS-NEXT: [[SHL2:%[0-9]+]]:sgpr(s64) = G_SHL [[MV3]], [[C9]](s32)
+ ; NEW_RBS-NEXT: [[COPY25:%[0-9]+]]:vgpr(s64) = COPY [[SHL2]](s64)
+ ; NEW_RBS-NEXT: [[PTR_ADD2:%[0-9]+]]:vgpr(p1) = G_PTR_ADD [[MV]], [[COPY25]](s64)
; NEW_RBS-NEXT: [[LOAD2:%[0-9]+]]:vgpr(s32) = G_LOAD [[PTR_ADD2]](p1) :: (load (s32), addrspace 1)
- ; NEW_RBS-NEXT: [[C9:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; NEW_RBS-NEXT: [[COPY20:%[0-9]+]]:vgpr(s32) = COPY [[C9]](s32)
- ; NEW_RBS-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[LOAD2]], [[COPY20]]
+ ; NEW_RBS-NEXT: [[C10:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+ ; NEW_RBS-NEXT: [[COPY26:%[0-9]+]]:vgpr(s32) = COPY [[C10]](s32)
+ ; NEW_RBS-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[LOAD2]], [[COPY26]]
; NEW_RBS-NEXT: G_STORE [[ADD]](s32), [[PTR_ADD2]](p1) :: (store (s32), addrspace 1)
- ; NEW_RBS-NEXT: [[ADD1:%[0-9]+]]:sgpr(s32) = G_ADD [[PHI2]], [[C9]]
- ; NEW_RBS-NEXT: [[C10:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 100
- ; NEW_RBS-NEXT: [[ICMP2:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ult), [[PHI2]](s32), [[C10]]
- ; NEW_RBS-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP2]], [[C9]]
- ; NEW_RBS-NEXT: [[AMDGPU_COPY_VCC_SCC2:%[0-9]+]]:sreg_32(s1) = G_AMDGPU_COPY_VCC_SCC [[AND]](s32)
- ; NEW_RBS-NEXT: [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY14]](s1), $exec_lo, implicit-def $scc
- ; NEW_RBS-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[AMDGPU_COPY_VCC_SCC2]](s1), implicit-def $scc
- ; NEW_RBS-NEXT: [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc
+ ; NEW_RBS-NEXT: [[ADD1:%[0-9]+]]:sgpr(s32) = G_ADD [[PHI3]], [[C10]]
+ ; NEW_RBS-NEXT: [[C11:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 100
+ ; NEW_RBS-NEXT: [[ICMP2:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ult), [[PHI3]](s32), [[C11]]
+ ; NEW_RBS-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP2]], [[C10]]
+ ; NEW_RBS-NEXT: [[AMDGPU_COPY_VCC_SCC3:%[0-9]+]]:sreg_32(s1) = G_AMDGPU_COPY_VCC_SCC [[AND]](s32)
+ ; NEW_RBS-NEXT: [[S_ANDN2_B32_3:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY16]](s1), $exec_lo, implicit-def $scc
+ ; NEW_RBS-NEXT: [[S_AND_B32_3:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[AMDGPU_COPY_VCC_SCC3]](s1), implicit-def $scc
+ ; NEW_RBS-NEXT: [[S_OR_B32_3:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_3]](s1), [[S_AND_B32_3]](s1), implicit-def $scc
; NEW_RBS-NEXT: {{ $}}
; NEW_RBS-NEXT: bb.5:
; NEW_RBS-NEXT: successors: %bb.3(0x80000000)
; NEW_RBS-NEXT: {{ $}}
- ; NEW_RBS-NEXT: [[PHI5:%[0-9]+]]:sreg_32(s1) = PHI [[AMDGPU_COPY_VCC_SCC1]](s1), %bb.2, [[S_OR_B32_1]](s1), %bb.4
- ; NEW_RBS-NEXT: [[PHI6:%[0-9]+]]:sgpr(s32) = G_PHI [[ADD1]](s32), %bb.4, [[DEF]](s32), %bb.2
- ; NEW_RBS-NEXT: [[COPY21:%[0-9]+]]:sreg_32(s1) = COPY [[PHI5]](s1)
- ; NEW_RBS-NEXT: [[COPY22:%[0-9]+]]:sreg_32(s1) = COPY [[COPY21]](s1)
- ; NEW_RBS-NEXT: [[COPY23:%[0-9]+]]:sgpr(s32) = COPY [[SI_IF1]](s32)
- ; NEW_RBS-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[COPY23]](s32)
- ; NEW_RBS-NEXT: [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY10]](s1), $exec_lo, implicit-def $scc
- ; NEW_RBS-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY22]](s1), implicit-def $scc
- ; NEW_RBS-NEXT: [[S_OR_B32_2:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc
+ ; NEW_RBS-NEXT: [[PHI7:%[0-9]+]]:sreg_32(s1) = PHI [[AMDGPU_COPY_VCC_SCC2]](s1), %bb.2, [[S_OR_B32_3]](s1), %bb.4
+ ; NEW_RBS-NEXT: [[PHI8:%[0-9]+]]:sgpr(s32) = G_PHI [[ADD1]](s32), %bb.4, [[DEF]](s32), %bb.2
+ ; NEW_RBS-NEXT: [[COPY27:%[0-9]+]]:sreg_32(s1) = COPY [[PHI7]](s1)
+ ; NEW_RBS-NEXT: [[COPY28:%[0-9]+]]:sreg_32(s1) = COPY [[COPY27]](s1)
+ ; NEW_RBS-NEXT: [[COPY29:%[0-9]+]]:vcc(s1) = COPY [[SI_IF1]](s1)
+ ; NEW_RBS-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[COPY29]](s1)
+ ; NEW_RBS-NEXT: [[S_ANDN2_B32_4:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY12]](s1), $exec_lo, implicit-def $scc
+ ; NEW_RBS-NEXT: [[S_AND_B32_4:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY28]](s1), implicit-def $scc
+ ; NEW_RBS-NEXT: [[S_OR_B32_4:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_4]](s1), [[S_AND_B32_4]](s1), implicit-def $scc
+ ; NEW_RBS-NEXT: [[DEF3:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
; NEW_RBS-NEXT: G_BR %bb.3
; NEW_RBS-NEXT: {{ $}}
; NEW_RBS-NEXT: bb.6:
- ; NEW_RBS-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
+ ; NEW_RBS-NEXT: [[COPY30:%[0-9]+]]:vcc(s1) = COPY [[S_OR_B32_1]](s1)
+ ; NEW_RBS-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[COPY30]](s1)
; NEW_RBS-NEXT: S_ENDPGM 0
bb.0:
successors: %bb.1(0x80000000)
@@ -1297,89 +1379,106 @@ body: |
%8:_(p1) = G_MERGE_VALUES %6(s32), %7(s32)
%9:_(s32) = G_IMPLICIT_DEF
%10:_(s32) = G_CONSTANT i32 0
- %11:sreg_32(s1) = IMPLICIT_DEF
+ %11:_(s1) = G_CONSTANT i1 false
+ %12:sreg_32(s1) = IMPLICIT_DEF
+ %66:sreg_32(s1) = IMPLICIT_DEF
+ %73:sreg_32(s1) = COPY %11(s1)
bb.1:
successors: %bb.2(0x40000000), %bb.3(0x40000000)
- %12:sreg_32(s1) = PHI %11(s1), %bb.0, %13(s1), %bb.3
- %14:_(s32) = G_PHI %15(s32), %bb.3, %10(s32), %bb.0
- %16:_(s32) = G_PHI %10(s32), %bb.0, %17(s32), %bb.3
- %18:sreg_32(s1) = COPY %12(s1)
- %19:_(s64) = G_SEXT %16(s32)
- %20:_(s32) = G_CONSTANT i32 2
- %21:_(s64) = G_SHL %19, %20(s32)
- %22:_(p1) = G_PTR_ADD %5, %21(s64)
- %23:_(s32) = G_LOAD %22(p1) :: (load (s32), addrspace 1)
- %24:_(s32) = G_CONSTANT i32 0
- %25:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %23(s32), %24
- %26:_(s1) = G_CONSTANT i1 true
- %27:sreg_32(s1) = COPY %26(s1)
- %28:sreg_32(s1) = S_ANDN2_B32 %18(s1), $exec_lo, implicit-def $scc
- %29:sreg_32(s1) = S_AND_B32 $exec_lo, %27(s1), implicit-def $scc
- %30:sreg_32(s1) = S_OR_B32 %28(s1), %29(s1), implicit-def $scc
- %31:sreg_32(s1) = COPY %30(s1)
- %32:sreg_32_xm0_xexec(s32) = SI_IF %25(s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+ %76:sreg_32(s1) = PHI %73(s1), %bb.0, %74(s1), %bb.3
+ %67:sreg_32(s1) = PHI %66(s1), %bb.0, %65(s1), %bb.3
+ %13:sreg_32(s1) = PHI %12(s1), %bb.0, %14(s1), %bb.3
+ %17:_(s32) = G_PHI %10(s32), %bb.0, %18(s32), %bb.3
+ %15:sreg_32(s1) = COPY %76(s1)
+ %68:sreg_32(s1) = COPY %67(s1)
+ %19:sreg_32(s1) = COPY %13(s1)
+ %20:_(s64) = G_SEXT %17(s32)
+ %21:_(s32) = G_CONSTANT i32 2
+ %22:_(s64) = G_SHL %20, %21(s32)
+ %23:_(p1) = G_PTR_ADD %5, %22(s64)
+ %24:_(s32) = G_LOAD %23(p1) :: (load (s32), addrspace 1)
+ %25:_(s32) = G_CONSTANT i32 0
+ %26:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %24(s32), %25
+ %27:_(s1) = G_CONSTANT i1 true
+ %28:sreg_32(s1) = COPY %27(s1)
+ %29:sreg_32(s1) = S_ANDN2_B32 %19(s1), $exec_lo, implicit-def $scc
+ %30:sreg_32(s1) = S_AND_B32 $exec_lo, %28(s1), implicit-def $scc
+ %31:sreg_32(s1) = S_OR_B32 %29(s1), %30(s1), implicit-def $scc
+ %32:sreg_32(s1) = COPY %31(s1)
+ %33:sreg_32_xm0_xexec(s1) = SI_IF %26(s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
G_BR %bb.2
bb.2:
successors: %bb.4(0x40000000), %bb.5(0x40000000)
- %33:_(s32) = G_CONSTANT i32 2
- %34:_(s64) = G_SHL %19, %33(s32)
- %35:_(p1) = G_PTR_ADD %8, %34(s64)
- %36:_(s32) = G_LOAD %35(p1) :: (load (s32), addrspace 1)
- %37:_(s32) = G_CONSTANT i32 0
- %38:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %36(s32), %37
- %39:_(s1) = G_CONSTANT i1 true
- %40:sreg_32(s1) = COPY %39(s1)
+ %34:_(s32) = G_CONSTANT i32 2
+ %35:_(s64) = G_SHL %20, %34(s32)
+ %36:_(p1) = G_PTR_ADD %8, %35(s64)
+ %37:_(s32) = G_LOAD %36(p1) :: (load (s32), addrspace 1)
+ %38:_(s32) = G_CONSTANT i32 0
+ %39:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %37(s32), %38
+ %40:_(s1) = G_CONSTANT i1 true
%41:sreg_32(s1) = COPY %40(s1)
- %42:sreg_32_xm0_xexec(s32) = SI_IF %38(s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+ %42:sreg_32(s1) = COPY %41(s1)
+ %43:sreg_32_xm0_xexec(s1) = SI_IF %39(s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
G_BR %bb.4
bb.3:
successors: %bb.6(0x04000000), %bb.1(0x7c000000)
- %13:sreg_32(s1) = PHI %30(s1), %bb.1, %43(s1), %bb.5
- %17:_(s32) = G_PHI %44(s32), %bb.5, %9(s32), %bb.1
- %45:sreg_32(s1) = COPY %13(s1)
- G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %32(s32)
- %15:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %45(s1), %14(s32)
- SI_LOOP %15(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+ %75:sreg_32(s1) = PHI %76(s1), %bb.1, %72(s1), %bb.5
+ %14:sreg_32(s1) = PHI %31(s1), %bb.1, %44(s1), %bb.5
+ %18:_(s32) = G_PHI %45(s32), %bb.5, %9(s32), %bb.1
+ %77:sreg_32(s1) = COPY %75(s1)
+ %46:sreg_32(s1) = COPY %14(s1)
+ G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %33(s1)
+ %16:sreg_32_xm0_xexec(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %46(s1), %15(s1)
+ %78:sreg_32(s1) = COPY %16(s1)
+ %69:sreg_32(s1) = COPY %16(s1)
+ %70:sreg_32(s1) = S_ANDN2_B32 %68(s1), $exec_lo, implicit-def $scc
+ %71:sreg_32(s1) = S_AND_B32 $exec_lo, %69(s1), implicit-def $scc
+ %65:sreg_32(s1) = S_OR_B32 %70(s1), %71(s1), implicit-def $scc
+ %79:sreg_32(s1) = S_ANDN2_B32 %77(s1), $exec_lo, implicit-def $scc
+ %80:sreg_32(s1) = S_AND_B32 $exec_lo, %78(s1), implicit-def $scc
+ %74:sreg_32(s1) = S_OR_B32 %79(s1), %80(s1), implicit-def $scc
+ SI_LOOP %16(s1), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
G_BR %bb.6
bb.4:
successors: %bb.5(0x80000000)
- %46:_(s32) = G_CONSTANT i32 2
- %47:_(s64) = G_SHL %19, %46(s32)
- %48:_(p1) = G_PTR_ADD %2, %47(s64)
- %49:_(s32) = G_LOAD %48(p1) :: (load (s32), addrspace 1)
- %50:_(s32) = G_CONSTANT i32 1
- %51:_(s32) = G_ADD %49, %50
- G_STORE %51(s32), %48(p1) :: (store (s32), addrspace 1)
- %52:_(s32) = G_ADD %16, %50
- %53:_(s32) = G_CONSTANT i32 100
- %54:_(s1) = G_ICMP intpred(ult), %16(s32), %53
- %55:sreg_32(s1) = COPY %54(s1)
- %56:sreg_32(s1) = S_ANDN2_B32 %41(s1), $exec_lo, implicit-def $scc
- %57:sreg_32(s1) = S_AND_B32 $exec_lo, %55(s1), implicit-def $scc
- %58:sreg_32(s1) = S_OR_B32 %56(s1), %57(s1), implicit-def $scc
+ %47:_(s32) = G_CONSTANT i32 2
+ %48:_(s64) = G_SHL %20, %47(s32)
+ %49:_(p1) = G_PTR_ADD %2, %48(s64)
+ %50:_(s32) = G_LOAD %49(p1) :: (load (s32), addrspace 1)
+ %51:_(s32) = G_CONSTANT i32 1
+ %52:_(s32) = G_ADD %50, %51
+ G_STORE %52(s32), %49(p1) :: (store (s32), addrspace 1)
+ %53:_(s32) = G_ADD %17, %51
+ %54:_(s32) = G_CONSTANT i32 100
+ %55:_(s1) = G_ICMP intpred(ult), %17(s32), %54
+ %56:sreg_32(s1) = COPY %55(s1)
+ %57:sreg_32(s1) = S_ANDN2_B32 %42(s1), $exec_lo, implicit-def $scc
+ %58:sreg_32(s1) = S_AND_B32 $exec_lo, %56(s1), implicit-def $scc
+ %59:sreg_32(s1) = S_OR_B32 %57(s1), %58(s1), implicit-def $scc
bb.5:
successors: %bb.3(0x80000000)
- %59:sreg_32(s1) = PHI %40(s1), %bb.2, %58(s1), %bb.4
- %44:_(s32) = G_PHI %52(s32), %bb.4, %9(s32), %bb.2
- %60:sreg_32(s1) = COPY %59(s1)
+ %60:sreg_32(s1) = PHI %41(s1), %bb.2, %59(s1), %bb.4
+ %45:_(s32) = G_PHI %53(s32), %bb.4, %9(s32), %bb.2
%61:sreg_32(s1) = COPY %60(s1)
- G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %42(s32)
- %62:sreg_32(s1) = S_ANDN2_B32 %31(s1), $exec_lo, implicit-def $scc
- %63:sreg_32(s1) = S_AND_B32 $exec_lo, %61(s1), implicit-def $scc
- %43:sreg_32(s1) = S_OR_B32 %62(s1), %63(s1), implicit-def $scc
+ %62:sreg_32(s1) = COPY %61(s1)
+ G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %43(s1)
+ %63:sreg_32(s1) = S_ANDN2_B32 %32(s1), $exec_lo, implicit-def $scc
+ %64:sreg_32(s1) = S_AND_B32 $exec_lo, %62(s1), implicit-def $scc
+ %44:sreg_32(s1) = S_OR_B32 %63(s1), %64(s1), implicit-def $scc
+ %72:sreg_32(s1) = IMPLICIT_DEF
G_BR %bb.3
bb.6:
- G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %15(s32)
+ G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %65(s1)
S_ENDPGM 0
...
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
index 7c8b2ce209434..ad86d3e2406b4 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
@@ -411,29 +411,33 @@ define void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, p
; GISEL-NEXT: global_load_dword v[0:1], off lds
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: global_load_dword v[0:1], off lds
-; GISEL-NEXT: s_mov_b32 s7, 0
-; GISEL-NEXT: s_mov_b32 s6, 2
+; GISEL-NEXT: s_mov_b32 s9, 0
+; GISEL-NEXT: s_mov_b32 s8, 2
; GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GISEL-NEXT: v_mov_b32_e32 v5, s7
+; GISEL-NEXT: v_mov_b32_e32 v5, s9
; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
; GISEL-NEXT: .LBB4_1: ; %loop_body
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GISEL-NEXT: v_readfirstlane_b32 s7, v2
-; GISEL-NEXT: s_mov_b32 m0, s7
-; GISEL-NEXT: s_add_i32 s6, s6, 1
+; GISEL-NEXT: v_readfirstlane_b32 s9, v2
+; GISEL-NEXT: s_mov_b32 m0, s9
+; GISEL-NEXT: s_add_i32 s8, s8, 1
; GISEL-NEXT: global_load_dword v[0:1], off lds
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; wait_asyncmark(2)
; GISEL-NEXT: s_waitcnt vmcnt(2)
; GISEL-NEXT: ds_read_b32 v6, v2
-; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, s6, v7
+; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, s8, v7
; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_add_u32_e32 v5, v5, v6
; GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GISEL-NEXT: s_cbranch_execnz .LBB4_1
; GISEL-NEXT: ; %bb.2: ; %epilog
-; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GISEL-NEXT: ; wait_asyncmark(1)
; GISEL-NEXT: s_waitcnt vmcnt(1)
; GISEL-NEXT: ds_read_b32 v0, v2
@@ -557,43 +561,47 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-NEXT: v_readfirstlane_b32 s4, v2
; GISEL-NEXT: s_mov_b32 m0, s4
-; GISEL-NEXT: global_load_dword v10, v[0:1], off
-; GISEL-NEXT: global_load_dword v14, v[3:4], off
-; GISEL-NEXT: s_mov_b32 s6, 2
+; GISEL-NEXT: global_load_dword v13, v[0:1], off
+; GISEL-NEXT: global_load_dword v15, v[3:4], off
+; GISEL-NEXT: s_mov_b32 s8, 2
; GISEL-NEXT: global_load_dword v[0:1], off lds
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: global_load_dword v8, v[0:1], off
; GISEL-NEXT: global_load_dword v9, v[3:4], off
; GISEL-NEXT: s_mov_b64 s[4:5], 0
; GISEL-NEXT: global_load_dword v[0:1], off lds
+; GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: s_waitcnt vmcnt(2)
-; GISEL-NEXT: v_mov_b32_e32 v13, v8
+; GISEL-NEXT: v_mov_b32_e32 v12, v8
; GISEL-NEXT: s_waitcnt vmcnt(1)
-; GISEL-NEXT: v_mov_b32_e32 v15, v9
+; GISEL-NEXT: v_mov_b32_e32 v14, v9
; GISEL-NEXT: .LBB5_1: ; %loop_body
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GISEL-NEXT: v_readfirstlane_b32 s7, v2
+; GISEL-NEXT: v_readfirstlane_b32 s9, v2
; GISEL-NEXT: s_waitcnt vmcnt(1)
-; GISEL-NEXT: v_mov_b32_e32 v12, v15
-; GISEL-NEXT: v_mov_b32_e32 v11, v13
-; GISEL-NEXT: global_load_dword v13, v[0:1], off
-; GISEL-NEXT: global_load_dword v15, v[3:4], off
-; GISEL-NEXT: s_mov_b32 m0, s7
-; GISEL-NEXT: s_add_i32 s6, s6, 1
+; GISEL-NEXT: v_mov_b32_e32 v11, v14
+; GISEL-NEXT: v_mov_b32_e32 v10, v12
+; GISEL-NEXT: global_load_dword v12, v[0:1], off
+; GISEL-NEXT: global_load_dword v14, v[3:4], off
+; GISEL-NEXT: s_mov_b32 m0, s9
+; GISEL-NEXT: s_add_i32 s8, s8, 1
; GISEL-NEXT: global_load_dword v[0:1], off lds
-; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, s6, v7
-; GISEL-NEXT: v_mov_b32_e32 v16, v14
-; GISEL-NEXT: v_mov_b32_e32 v17, v10
-; GISEL-NEXT: v_mov_b32_e32 v10, v8
-; GISEL-NEXT: v_mov_b32_e32 v14, v9
+; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, s8, v7
; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GISEL-NEXT: v_mov_b32_e32 v16, v15
+; GISEL-NEXT: v_mov_b32_e32 v17, v13
+; GISEL-NEXT: v_mov_b32_e32 v13, v8
+; GISEL-NEXT: v_mov_b32_e32 v15, v9
+; GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; wait_asyncmark(2)
; GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GISEL-NEXT: s_cbranch_execnz .LBB5_1
; GISEL-NEXT: ; %bb.2: ; %epilog
-; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GISEL-NEXT: ds_read_b32 v0, v2
; GISEL-NEXT: ; wait_asyncmark(1)
; GISEL-NEXT: s_waitcnt vmcnt(3)
@@ -603,10 +611,10 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
; GISEL-NEXT: ds_read_b32 v2, v2
; GISEL-NEXT: v_add_u32_e32 v3, v17, v16
; GISEL-NEXT: s_waitcnt lgkmcnt(2)
-; GISEL-NEXT: v_add3_u32 v0, v3, v0, v12
+; GISEL-NEXT: v_add3_u32 v0, v3, v0, v11
; GISEL-NEXT: s_waitcnt lgkmcnt(1)
-; GISEL-NEXT: v_add3_u32 v0, v11, v0, v1
-; GISEL-NEXT: v_add_u32_e32 v1, v13, v15
+; GISEL-NEXT: v_add3_u32 v0, v10, v0, v1
+; GISEL-NEXT: v_add_u32_e32 v1, v12, v14
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_add3_u32 v0, v1, v2, v0
; GISEL-NEXT: global_store_dword v[5:6], v0, off
diff --git a/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_cond.ll b/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_cond.ll
index 292b3de7ca71e..dfbfe77e56d5f 100644
--- a/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_cond.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_cond.ll
@@ -50,7 +50,8 @@ define amdgpu_kernel void @flat_atomic_usub_cond_no_rtn_u32(ptr %addr, i32 %in)
; GFX9-GISEL-LABEL: flat_atomic_usub_cond_no_rtn_u32:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr2_sgpr3
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: s_add_u32 s0, s0, -16
; GFX9-GISEL-NEXT: s_addc_u32 s1, s1, -1
@@ -61,15 +62,18 @@ define amdgpu_kernel void @flat_atomic_usub_cond_no_rtn_u32(ptr %addr, i32 %in)
; GFX9-GISEL-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_subrev_u32_e32 v2, s2, v3
-; GFX9-GISEL-NEXT: v_cmp_le_u32_e32 vcc, s2, v3
+; GFX9-GISEL-NEXT: v_subrev_u32_e32 v2, s6, v3
+; GFX9-GISEL-NEXT: v_cmp_le_u32_e32 vcc, s6, v3
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
; GFX9-GISEL-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX9-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[4:5], exec, s[0:1]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-GISEL-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB0_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -156,7 +160,8 @@ define amdgpu_kernel void @flat_atomic_usub_cond_rtn_u32(ptr %addr, i32 %in, ptr
; GFX9-GISEL-LABEL: flat_atomic_usub_cond_rtn_u32:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-GISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dword s8, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: s_add_u32 s2, s0, 16
; GFX9-GISEL-NEXT: s_addc_u32 s3, s1, 0
@@ -168,22 +173,25 @@ define amdgpu_kernel void @flat_atomic_usub_cond_rtn_u32(ptr %addr, i32 %in, ptr
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-GISEL-NEXT: s_add_u32 s8, s0, 16
-; GFX9-GISEL-NEXT: v_subrev_u32_e32 v0, s6, v1
-; GFX9-GISEL-NEXT: s_addc_u32 s9, s1, 0
-; GFX9-GISEL-NEXT: v_cmp_le_u32_e32 vcc, s6, v1
+; GFX9-GISEL-NEXT: s_add_u32 s10, s0, 16
+; GFX9-GISEL-NEXT: v_subrev_u32_e32 v0, s8, v1
+; GFX9-GISEL-NEXT: s_addc_u32 s11, s1, 0
+; GFX9-GISEL-NEXT: v_cmp_le_u32_e32 vcc, s8, v1
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, s9
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s10
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, s11
; GFX9-GISEL-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB1_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s1
@@ -263,13 +271,14 @@ define amdgpu_kernel void @global_atomic_usub_cond_no_rtn_u32(ptr addrspace(1) %
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-GISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr4_sgpr5
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: s_add_u32 s2, s0, -16
; GFX9-GISEL-NEXT: s_addc_u32 s3, s1, -1
-; GFX9-GISEL-NEXT: s_load_dword s4, s[2:3], 0x0
+; GFX9-GISEL-NEXT: s_load_dword s7, s[2:3], 0x0
; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s4
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s7
; GFX9-GISEL-NEXT: .LBB2_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: v_subrev_u32_e32 v0, s6, v1
@@ -280,7 +289,10 @@ define amdgpu_kernel void @global_atomic_usub_cond_no_rtn_u32(ptr addrspace(1) %
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[8:9], exec, s[2:3]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB2_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -353,28 +365,32 @@ define amdgpu_kernel void @global_atomic_usub_cond_rtn_u32(ptr addrspace(1) %add
; GFX9-GISEL-LABEL: global_atomic_usub_cond_rtn_u32:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-GISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dword s8, s[4:5], 0x2c
; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_load_dword s7, s[0:1], 0x10
+; GFX9-GISEL-NEXT: s_load_dword s9, s[0:1], 0x10
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s9
; GFX9-GISEL-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v1
-; GFX9-GISEL-NEXT: v_subrev_u32_e32 v1, s6, v2
-; GFX9-GISEL-NEXT: v_cmp_le_u32_e32 vcc, s6, v2
+; GFX9-GISEL-NEXT: v_subrev_u32_e32 v1, s8, v2
+; GFX9-GISEL-NEXT: v_cmp_le_u32_e32 vcc, s8, v2
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
; GFX9-GISEL-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] offset:16 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB3_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -443,6 +459,7 @@ define amdgpu_kernel void @ds_usub_cond_no_rtn_u32(ptr addrspace(3) %addr, i32 %
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr4_sgpr5
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: s_add_u32 s0, s0, -16
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
@@ -459,7 +476,10 @@ define amdgpu_kernel void @ds_usub_cond_no_rtn_u32(ptr addrspace(3) %addr, i32 %
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[6:7], exec, s[2:3]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB4_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -526,6 +546,7 @@ define amdgpu_kernel void @ds_usub_cond_rtn_u32(ptr addrspace(3) %addr, i32 %in,
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-GISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr4_sgpr5
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX9-GISEL-NEXT: ds_read_b32 v1, v0 offset:16
@@ -542,10 +563,13 @@ define amdgpu_kernel void @ds_usub_cond_rtn_u32(ptr addrspace(3) %addr, i32 %in,
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[8:9], exec, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB5_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s6
; GFX9-GISEL-NEXT: ds_write_b32 v0, v1
; GFX9-GISEL-NEXT: s_endpgm
@@ -611,6 +635,7 @@ define i32 @global_atomic_usub_cond(ptr addrspace(1) %ptr, i32 %data) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: global_load_dword v3, v[0:1], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX9-GISEL-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -623,10 +648,13 @@ define i32 @global_atomic_usub_cond(ptr addrspace(1) %ptr, i32 %data) {
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB6_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -695,6 +723,7 @@ define i32 @global_atomic_usub_cond_offset(ptr addrspace(1) %ptr, i32 %data) {
; GFX9-GISEL-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v1, vcc
; GFX9-GISEL-NEXT: global_load_dword v0, v[3:4], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX9-GISEL-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -707,10 +736,13 @@ define i32 @global_atomic_usub_cond_offset(ptr addrspace(1) %ptr, i32 %data) {
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB7_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: global_atomic_usub_cond_offset:
@@ -772,6 +804,7 @@ define void @global_atomic_usub_cond_nortn(ptr addrspace(1) %ptr, i32 %data) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: global_load_dword v4, v[0:1], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX9-GISEL-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -783,11 +816,14 @@ define void @global_atomic_usub_cond_nortn(ptr addrspace(1) %ptr, i32 %data) {
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB8_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: global_atomic_usub_cond_nortn:
@@ -855,6 +891,7 @@ define void @global_atomic_usub_cond_offset_nortn(ptr addrspace(1) %ptr, i32 %da
; GFX9-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-GISEL-NEXT: global_load_dword v4, v[0:1], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX9-GISEL-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -866,11 +903,14 @@ define void @global_atomic_usub_cond_offset_nortn(ptr addrspace(1) %ptr, i32 %da
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[8:9], exec, s[4:5]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB9_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: global_atomic_usub_cond_offset_nortn:
@@ -940,28 +980,32 @@ define amdgpu_kernel void @global_atomic_usub_cond_sgpr_base_offset(ptr addrspac
; GFX9-GISEL-LABEL: global_atomic_usub_cond_sgpr_base_offset:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-GISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dword s8, s[4:5], 0x2c
; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x1000
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_load_dword s7, s[0:1], 0x1000
+; GFX9-GISEL-NEXT: s_load_dword s9, s[0:1], 0x1000
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s9
; GFX9-GISEL-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v1
-; GFX9-GISEL-NEXT: v_subrev_u32_e32 v1, s6, v2
-; GFX9-GISEL-NEXT: v_cmp_le_u32_e32 vcc, s6, v2
+; GFX9-GISEL-NEXT: v_subrev_u32_e32 v1, s8, v2
+; GFX9-GISEL-NEXT: v_cmp_le_u32_e32 vcc, s8, v2
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
; GFX9-GISEL-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB10_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -1031,10 +1075,11 @@ define amdgpu_kernel void @global_atomic_usub_cond_sgpr_base_offset_nortn(ptr ad
; GFX9-GISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x1000
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr4_sgpr5
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_load_dword s4, s[0:1], 0x1000
+; GFX9-GISEL-NEXT: s_load_dword s7, s[0:1], 0x1000
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s4
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s7
; GFX9-GISEL-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: v_subrev_u32_e32 v0, s6, v1
@@ -1045,7 +1090,10 @@ define amdgpu_kernel void @global_atomic_usub_cond_sgpr_base_offset_nortn(ptr ad
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[8:9], exec, s[2:3]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB11_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1108,6 +1156,7 @@ define i32 @global_atomic_usub_cond__amdgpu_no_remote_memory(ptr addrspace(1) %p
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: global_load_dword v3, v[0:1], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX9-GISEL-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -1120,10 +1169,13 @@ define i32 @global_atomic_usub_cond__amdgpu_no_remote_memory(ptr addrspace(1) %p
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB12_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -1186,6 +1238,7 @@ define i32 @global_atomic_usub_cond__amdgpu_no_fine_grained_memory(ptr addrspace
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: global_load_dword v3, v[0:1], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX9-GISEL-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -1198,10 +1251,13 @@ define i32 @global_atomic_usub_cond__amdgpu_no_fine_grained_memory(ptr addrspace
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB13_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -1264,6 +1320,7 @@ define i32 @global_atomic_usub_cond__amdgpu_no_fine_grained_memory__amdgpu_no_re
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: global_load_dword v3, v[0:1], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX9-GISEL-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -1276,10 +1333,13 @@ define i32 @global_atomic_usub_cond__amdgpu_no_fine_grained_memory__amdgpu_no_re
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB14_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll b/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll
index 04c78cd3cb0d1..bb970fa05cb7d 100644
--- a/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll
@@ -14,6 +14,8 @@ define i32 @global_atomic_usub_sat(ptr addrspace(1) %ptr, i32 %data) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: global_load_dword v3, v[0:1], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
; GFX9-GISEL-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -24,10 +26,16 @@ define i32 @global_atomic_usub_sat(ptr addrspace(1) %ptr, i32 %data) {
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB0_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -127,12 +135,11 @@ define i32 @global_atomic_usub_sat_offset(ptr addrspace(1) %ptr, i32 %data) {
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_add_co_u32_e32 v3, vcc, 0x1000, v0
-; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], vcc
-; GFX9-GISEL-NEXT: v_addc_co_u32_e64 v4, s[4:5], 0, v1, s[4:5]
-; GFX9-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v3
-; GFX9-GISEL-NEXT: global_load_dword v0, v[0:1], off
+; GFX9-GISEL-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v1, vcc
+; GFX9-GISEL-NEXT: global_load_dword v0, v[3:4], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
; GFX9-GISEL-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -143,10 +150,16 @@ define i32 @global_atomic_usub_sat_offset(ptr addrspace(1) %ptr, i32 %data) {
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB1_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: global_atomic_usub_sat_offset:
@@ -261,6 +274,8 @@ define void @global_atomic_usub_sat_nortn(ptr addrspace(1) %ptr, i32 %data) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: global_load_dword v4, v[0:1], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
; GFX9-GISEL-NEXT: .LBB2_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -270,11 +285,17 @@ define void @global_atomic_usub_sat_nortn(ptr addrspace(1) %ptr, i32 %data) {
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB2_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: global_atomic_usub_sat_nortn:
@@ -371,27 +392,32 @@ define void @global_atomic_usub_sat_offset_nortn(ptr addrspace(1) %ptr, i32 %dat
; GFX9-GISEL-LABEL: global_atomic_usub_sat_offset_nortn:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_add_co_u32_e32 v3, vcc, 0x1000, v0
-; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], vcc
-; GFX9-GISEL-NEXT: v_addc_co_u32_e64 v4, s[4:5], 0, v1, s[4:5]
+; GFX9-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0x1000, v0
; GFX9-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v3
-; GFX9-GISEL-NEXT: global_load_dword v1, v[0:1], off
+; GFX9-GISEL-NEXT: global_load_dword v4, v[0:1], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
; GFX9-GISEL-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_sub_u32_e64 v0, v1, v2 clamp
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v[3:4], v[0:1], off glc
+; GFX9-GISEL-NEXT: v_sub_u32_e64 v3, v4, v2 clamp
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB3_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: global_atomic_usub_sat_offset_nortn:
@@ -503,33 +529,39 @@ define void @global_atomic_usub_sat_offset_nortn(ptr addrspace(1) %ptr, i32 %dat
define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset(ptr addrspace(1) %ptr, i32 %data, ptr addrspace(1) %dst) {
; GFX9-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x0
-; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX9-GISEL-NEXT: s_mov_b64 s[0:1], 0
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-GISEL-NEXT: s_load_dword s10, s[8:9], 0x8
+; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x1000
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr4_sgpr5
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_load_dword s5, s[2:3], 0x1000
-; GFX9-GISEL-NEXT: s_add_u32 s2, s2, 0x1000
-; GFX9-GISEL-NEXT: s_addc_u32 s3, s3, 0
+; GFX9-GISEL-NEXT: s_load_dword s11, s[0:1], 0x1000
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s5
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s11
; GFX9-GISEL-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-GISEL-NEXT: v_sub_u32_e64 v2, v3, s4 clamp
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v1, v[2:3], s[2:3] glc
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-GISEL-NEXT: v_sub_u32_e64 v1, v2, s10 clamp
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
-; GFX9-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[12:13], exec, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[12:13]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[12:13], exec, s[6:7]
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[12:13]
+; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB4_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset:
@@ -663,26 +695,32 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset(ptr addrspace
define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn(ptr addrspace(1) %ptr, i32 %data) {
; GFX9-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x0
-; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX9-GISEL-NEXT: s_mov_b64 s[0:1], 0
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-GISEL-NEXT: s_load_dword s10, s[8:9], 0x8
+; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x1000
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr4_sgpr5
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_load_dword s5, s[2:3], 0x1000
-; GFX9-GISEL-NEXT: s_add_u32 s2, s2, 0x1000
-; GFX9-GISEL-NEXT: s_addc_u32 s3, s3, 0
+; GFX9-GISEL-NEXT: s_load_dword s8, s[0:1], 0x1000
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s8
; GFX9-GISEL-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-GISEL-NEXT: v_sub_u32_e64 v0, v1, s4 clamp
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[2:3] glc
+; GFX9-GISEL-NEXT: v_sub_u32_e64 v0, v1, s10 clamp
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX9-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[8:9], exec, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[8:9], exec, s[6:7]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
+; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB5_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-GISEL-NEXT: s_endpgm
@@ -797,22 +835,30 @@ define i16 @global_atomic_usub_sat_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: global_load_dword v3, v[0:1], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-GISEL-NEXT: s_mov_b32 s6, 0xffff0000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0xffff0000
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
; GFX9-GISEL-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
-; GFX9-GISEL-NEXT: v_sub_u16_e64 v3, v4, v2 clamp
-; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, s6, v3
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v6, v3
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v3, v6, v2 clamp
+; GFX9-GISEL-NEXT: v_and_or_b32 v5, v6, v4, v3
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB6_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -821,6 +867,9 @@ define i16 @global_atomic_usub_sat_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: global_load_dword v3, v[0:1], off
; GFX10-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX10-GISEL-NEXT: .p2align 6
; GFX10-GISEL-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -835,10 +884,16 @@ define i16 @global_atomic_usub_sat_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX10-GISEL-NEXT: buffer_gl0_inv
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX10-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-GISEL-NEXT: s_andn2_b32 s6, s6, exec_lo
+; GFX10-GISEL-NEXT: s_and_b32 s7, exec_lo, s4
+; GFX10-GISEL-NEXT: s_andn2_b32 s5, s5, exec_lo
+; GFX10-GISEL-NEXT: s_or_b32 s6, s6, s7
+; GFX10-GISEL-NEXT: s_and_b32 s7, exec_lo, s6
+; GFX10-GISEL-NEXT: s_or_b32 s5, s5, s7
; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB6_1
; GFX10-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -847,6 +902,9 @@ define i16 @global_atomic_usub_sat_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: global_load_b32 v3, v[0:1], off
; GFX11-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr2
+; GFX11-GISEL-NEXT: .p2align 6
; GFX11-GISEL-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -863,12 +921,18 @@ define i16 @global_atomic_usub_sat_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX11-GISEL-NEXT: buffer_gl0_inv
; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX11-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_not1_b32 s2, s2, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s3, exec_lo, s0
+; GFX11-GISEL-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-GISEL-NEXT: s_or_b32 s2, s2, s3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_b32 s3, exec_lo, s2
+; GFX11-GISEL-NEXT: s_or_b32 s1, s1, s3
; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB6_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-GISEL-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: global_atomic_usub_sat_16:
@@ -880,6 +944,8 @@ define i16 @global_atomic_usub_sat_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: global_load_b32 v3, v[0:1], off
; GFX12-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr2
; GFX12-GISEL-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
@@ -896,12 +962,22 @@ define i16 @global_atomic_usub_sat_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX12-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-GISEL-NEXT: s_and_not1_b32 s2, s2, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s3, exec_lo, s0
+; GFX12-GISEL-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 s2, s2, s3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s3, exec_lo, s2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 s1, s1, s3
; GFX12-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB6_1
; GFX12-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: global_atomic_usub_sat_16:
@@ -1025,22 +1101,30 @@ define i16 @global_atomic_usub_sat_offset_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: global_load_dword v3, v[0:1], off offset:2048
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-GISEL-NEXT: s_mov_b32 s6, 0xffff0000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0xffff0000
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
; GFX9-GISEL-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
-; GFX9-GISEL-NEXT: v_sub_u16_e64 v3, v4, v2 clamp
-; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, s6, v3
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2048 glc
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v6, v3
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v3, v6, v2 clamp
+; GFX9-GISEL-NEXT: v_and_or_b32 v5, v6, v4, v3
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2048 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB7_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -1050,7 +1134,10 @@ define i16 @global_atomic_usub_sat_offset_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX10-GISEL-NEXT: v_add_co_u32 v3, vcc_lo, 0x800, v0
; GFX10-GISEL-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX10-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr6
; GFX10-GISEL-NEXT: global_load_dword v0, v[3:4], off
+; GFX10-GISEL-NEXT: .p2align 6
; GFX10-GISEL-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -1065,10 +1152,16 @@ define i16 @global_atomic_usub_sat_offset_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX10-GISEL-NEXT: buffer_gl0_inv
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX10-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-GISEL-NEXT: s_andn2_b32 s6, s6, exec_lo
+; GFX10-GISEL-NEXT: s_and_b32 s7, exec_lo, s4
+; GFX10-GISEL-NEXT: s_andn2_b32 s5, s5, exec_lo
+; GFX10-GISEL-NEXT: s_or_b32 s6, s6, s7
+; GFX10-GISEL-NEXT: s_and_b32 s7, exec_lo, s6
+; GFX10-GISEL-NEXT: s_or_b32 s5, s5, s7
; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB7_1
; GFX10-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: global_atomic_usub_sat_offset_16:
@@ -1076,6 +1169,9 @@ define i16 @global_atomic_usub_sat_offset_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: global_load_b32 v3, v[0:1], off offset:2048
; GFX11-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr2
+; GFX11-GISEL-NEXT: .p2align 6
; GFX11-GISEL-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -1092,12 +1188,18 @@ define i16 @global_atomic_usub_sat_offset_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX11-GISEL-NEXT: buffer_gl0_inv
; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX11-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_not1_b32 s2, s2, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s3, exec_lo, s0
+; GFX11-GISEL-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-GISEL-NEXT: s_or_b32 s2, s2, s3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_b32 s3, exec_lo, s2
+; GFX11-GISEL-NEXT: s_or_b32 s1, s1, s3
; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB7_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-GISEL-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: global_atomic_usub_sat_offset_16:
@@ -1109,6 +1211,8 @@ define i16 @global_atomic_usub_sat_offset_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: global_load_b32 v3, v[0:1], off offset:2048
; GFX12-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr2
; GFX12-GISEL-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
@@ -1125,12 +1229,22 @@ define i16 @global_atomic_usub_sat_offset_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX12-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-GISEL-NEXT: s_and_not1_b32 s2, s2, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s3, exec_lo, s0
+; GFX12-GISEL-NEXT: s_and_not1_b32 s1, s1, exec_lo
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 s2, s2, s3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s3, exec_lo, s2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 s1, s1, s3
; GFX12-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB7_1
; GFX12-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: global_atomic_usub_sat_offset_16:
@@ -1256,22 +1370,30 @@ define void @global_atomic_usub_sat_nortn_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: global_load_dword v4, v[0:1], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-GISEL-NEXT: s_mov_b32 s6, 0xffff0000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 0xffff0000
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
; GFX9-GISEL-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_sub_u16_e64 v3, v4, v2 clamp
-; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, s6, v3
+; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, v5, v3
; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB8_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: global_atomic_usub_sat_nortn_16:
@@ -1279,6 +1401,9 @@ define void @global_atomic_usub_sat_nortn_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: global_load_dword v4, v[0:1], off
; GFX10-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX10-GISEL-NEXT: .p2align 6
; GFX10-GISEL-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -1293,10 +1418,16 @@ define void @global_atomic_usub_sat_nortn_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, v3
; GFX10-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-GISEL-NEXT: s_andn2_b32 s6, s6, exec_lo
+; GFX10-GISEL-NEXT: s_and_b32 s7, exec_lo, s4
+; GFX10-GISEL-NEXT: s_andn2_b32 s5, s5, exec_lo
+; GFX10-GISEL-NEXT: s_or_b32 s6, s6, s7
+; GFX10-GISEL-NEXT: s_and_b32 s7, exec_lo, s6
+; GFX10-GISEL-NEXT: s_or_b32 s5, s5, s7
; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB8_1
; GFX10-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: global_atomic_usub_sat_nortn_16:
@@ -1304,6 +1435,9 @@ define void @global_atomic_usub_sat_nortn_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: global_load_b32 v4, v[0:1], off
; GFX11-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr2
+; GFX11-GISEL-NEXT: .p2align 6
; GFX11-GISEL-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -1319,11 +1453,17 @@ define void @global_atomic_usub_sat_nortn_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX11-GISEL-NEXT: v_mov_b32_e32 v4, v3
; GFX11-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_not1_b32 s2, s2, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s3, exec_lo, s0
+; GFX11-GISEL-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-GISEL-NEXT: s_or_b32 s2, s2, s3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_b32 s3, exec_lo, s2
+; GFX11-GISEL-NEXT: s_or_b32 s1, s1, s3
; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: global_atomic_usub_sat_nortn_16:
@@ -1335,6 +1475,8 @@ define void @global_atomic_usub_sat_nortn_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: global_load_b32 v4, v[0:1], off
; GFX12-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr2
; GFX12-GISEL-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
@@ -1350,11 +1492,21 @@ define void @global_atomic_usub_sat_nortn_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX12-GISEL-NEXT: v_mov_b32_e32 v4, v3
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-GISEL-NEXT: s_and_not1_b32 s2, s2, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s3, exec_lo, s0
+; GFX12-GISEL-NEXT: s_and_not1_b32 s1, s1, exec_lo
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 s2, s2, s3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s3, exec_lo, s2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 s1, s1, s3
; GFX12-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB8_1
; GFX12-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: global_atomic_usub_sat_nortn_16:
@@ -1472,22 +1624,30 @@ define void @global_atomic_usub_sat_offset_nortn_16(ptr addrspace(1) %ptr, i16 %
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: global_load_dword v4, v[0:1], off offset:2048
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-GISEL-NEXT: s_mov_b32 s6, 0xffff0000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 0xffff0000
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
; GFX9-GISEL-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_sub_u16_e64 v3, v4, v2 clamp
-; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, s6, v3
+; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, v5, v3
; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2048 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB9_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: global_atomic_usub_sat_offset_nortn_16:
@@ -1496,7 +1656,10 @@ define void @global_atomic_usub_sat_offset_nortn_16(ptr addrspace(1) %ptr, i16 %
; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0x800, v0
; GFX10-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; GFX10-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr6
; GFX10-GISEL-NEXT: global_load_dword v4, v[0:1], off
+; GFX10-GISEL-NEXT: .p2align 6
; GFX10-GISEL-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -1511,10 +1674,16 @@ define void @global_atomic_usub_sat_offset_nortn_16(ptr addrspace(1) %ptr, i16 %
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, v3
; GFX10-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-GISEL-NEXT: s_andn2_b32 s6, s6, exec_lo
+; GFX10-GISEL-NEXT: s_and_b32 s7, exec_lo, s4
+; GFX10-GISEL-NEXT: s_andn2_b32 s5, s5, exec_lo
+; GFX10-GISEL-NEXT: s_or_b32 s6, s6, s7
+; GFX10-GISEL-NEXT: s_and_b32 s7, exec_lo, s6
+; GFX10-GISEL-NEXT: s_or_b32 s5, s5, s7
; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB9_1
; GFX10-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: global_atomic_usub_sat_offset_nortn_16:
@@ -1522,6 +1691,9 @@ define void @global_atomic_usub_sat_offset_nortn_16(ptr addrspace(1) %ptr, i16 %
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: global_load_b32 v4, v[0:1], off offset:2048
; GFX11-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr2
+; GFX11-GISEL-NEXT: .p2align 6
; GFX11-GISEL-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -1537,11 +1709,17 @@ define void @global_atomic_usub_sat_offset_nortn_16(ptr addrspace(1) %ptr, i16 %
; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX11-GISEL-NEXT: v_mov_b32_e32 v4, v3
; GFX11-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_not1_b32 s2, s2, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s3, exec_lo, s0
+; GFX11-GISEL-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-GISEL-NEXT: s_or_b32 s2, s2, s3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_b32 s3, exec_lo, s2
+; GFX11-GISEL-NEXT: s_or_b32 s1, s1, s3
; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: global_atomic_usub_sat_offset_nortn_16:
@@ -1553,6 +1731,8 @@ define void @global_atomic_usub_sat_offset_nortn_16(ptr addrspace(1) %ptr, i16 %
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: global_load_b32 v4, v[0:1], off offset:2048
; GFX12-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr2
; GFX12-GISEL-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
@@ -1568,11 +1748,21 @@ define void @global_atomic_usub_sat_offset_nortn_16(ptr addrspace(1) %ptr, i16 %
; GFX12-GISEL-NEXT: v_mov_b32_e32 v4, v3
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-GISEL-NEXT: s_and_not1_b32 s2, s2, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s3, exec_lo, s0
+; GFX12-GISEL-NEXT: s_and_not1_b32 s1, s1, exec_lo
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 s2, s2, s3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s3, exec_lo, s2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 s1, s1, s3
; GFX12-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB9_1
; GFX12-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: global_atomic_usub_sat_offset_nortn_16:
@@ -1691,32 +1881,40 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
; GFX9-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
+; GFX9-GISEL-NEXT: s_load_dword s10, s[8:9], 0x8
; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-GISEL-NEXT: s_mov_b32 s5, 0xffff0000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0xffff0000
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_load_dword s6, s[0:1], 0x800
+; GFX9-GISEL-NEXT: s_load_dword s11, s[0:1], 0x800
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr4_sgpr5
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s11
; GFX9-GISEL-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-GISEL-NEXT: v_sub_u16_e64 v0, v3, s4 clamp
-; GFX9-GISEL-NEXT: v_and_or_b32 v2, v3, s5, v0
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v1, v[2:3], s[0:1] offset:2048 glc
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v2, v3, s10 clamp
+; GFX9-GISEL-NEXT: v_and_or_b32 v2, v3, v0, v2
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v2, v1, v[2:3], s[0:1] offset:2048 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[12:13], exec, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[12:13]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[12:13], exec, s[6:7]
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[12:13]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB10_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_store_short v1, v0, s[0:1]
+; GFX9-GISEL-NEXT: global_store_short v0, v2, s[0:1]
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_16:
@@ -1724,14 +1922,15 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
; GFX10-GISEL-NEXT: s_clause 0x1
; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX10-GISEL-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0x800
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr4
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr5
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: s_load_dword s3, s[0:1], 0x800
-; GFX10-GISEL-NEXT: s_add_u32 s0, s0, 0x800
-; GFX10-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX10-GISEL-NEXT: s_mov_b32 s3, 0
+; GFX10-GISEL-NEXT: .p2align 6
; GFX10-GISEL-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, v1
@@ -1744,10 +1943,16 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
; GFX10-GISEL-NEXT: buffer_gl0_inv
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX10-GISEL-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX10-GISEL-NEXT: s_andn2_b32 s5, s5, exec_lo
+; GFX10-GISEL-NEXT: s_and_b32 s6, exec_lo, s3
+; GFX10-GISEL-NEXT: s_andn2_b32 s4, s4, exec_lo
+; GFX10-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX10-GISEL-NEXT: s_and_b32 s6, exec_lo, s5
+; GFX10-GISEL-NEXT: s_or_b32 s4, s4, s6
; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB10_1
; GFX10-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -1759,12 +1964,14 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
; GFX11-GISEL-NEXT: s_clause 0x1
; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr7
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: s_load_b32 s3, s[0:1], 0x800
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s3
; GFX11-GISEL-NEXT: s_mov_b32 s3, 0
+; GFX11-GISEL-NEXT: .p2align 6
; GFX11-GISEL-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -1779,11 +1986,17 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
; GFX11-GISEL-NEXT: buffer_gl0_inv
; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX11-GISEL-NEXT: s_or_b32 s3, vcc_lo, s3
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_not1_b32 s7, s7, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s8, exec_lo, s3
+; GFX11-GISEL-NEXT: s_and_not1_b32 s6, s6, exec_lo
+; GFX11-GISEL-NEXT: s_or_b32 s7, s7, s8
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_b32 s8, exec_lo, s7
+; GFX11-GISEL-NEXT: s_or_b32 s6, s6, s8
; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s6
; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x10
; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -1793,11 +2006,12 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
; GFX12-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_16:
; GFX12-GISEL: ; %bb.0:
; GFX12-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr7
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_load_b32 s3, s[0:1], 0x800
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s3
; GFX12-GISEL-NEXT: s_mov_b32 s3, 0
; GFX12-GISEL-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -1813,11 +2027,18 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
; GFX12-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX12-GISEL-NEXT: s_and_not1_b32 s7, s7, exec_lo
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s8, exec_lo, s3
+; GFX12-GISEL-NEXT: s_and_not1_b32 s6, s6, exec_lo
+; GFX12-GISEL-NEXT: s_or_b32 s7, s7, s8
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: s_and_b32 s8, exec_lo, s7
+; GFX12-GISEL-NEXT: s_or_b32 s6, s6, s8
; GFX12-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB10_1
; GFX12-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s6
; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x10
; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
@@ -1969,24 +2190,32 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_16(ptr
; GFX9-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn_16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
+; GFX9-GISEL-NEXT: s_load_dword s10, s[8:9], 0x8
; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-GISEL-NEXT: s_mov_b32 s5, 0xffff0000
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0xffff0000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_load_dword s6, s[0:1], 0x800
+; GFX9-GISEL-NEXT: s_load_dword s8, s[0:1], 0x800
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr4_sgpr5
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s8
; GFX9-GISEL-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-GISEL-NEXT: v_sub_u16_e64 v0, v1, s4 clamp
-; GFX9-GISEL-NEXT: v_and_or_b32 v0, v1, s5, v0
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:2048 glc
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v0, v1, s10 clamp
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v1, v2, v0
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] offset:2048 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[8:9], exec, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[8:9], exec, s[6:7]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB11_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1997,14 +2226,15 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_16(ptr
; GFX10-GISEL-NEXT: s_clause 0x1
; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX10-GISEL-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0x800
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr4
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr5
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: s_load_dword s3, s[0:1], 0x800
-; GFX10-GISEL-NEXT: s_add_u32 s0, s0, 0x800
-; GFX10-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX10-GISEL-NEXT: s_mov_b32 s3, 0
+; GFX10-GISEL-NEXT: .p2align 6
; GFX10-GISEL-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-GISEL-NEXT: v_sub_nc_u16 v0, v1, s2 clamp
@@ -2017,6 +2247,12 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_16(ptr
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, v0
; GFX10-GISEL-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX10-GISEL-NEXT: s_andn2_b32 s5, s5, exec_lo
+; GFX10-GISEL-NEXT: s_and_b32 s6, exec_lo, s3
+; GFX10-GISEL-NEXT: s_andn2_b32 s4, s4, exec_lo
+; GFX10-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX10-GISEL-NEXT: s_and_b32 s6, exec_lo, s5
+; GFX10-GISEL-NEXT: s_or_b32 s4, s4, s6
; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB11_1
; GFX10-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2027,12 +2263,14 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_16(ptr
; GFX11-GISEL-NEXT: s_clause 0x1
; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr4
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: s_load_b32 s3, s[0:1], 0x800
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
; GFX11-GISEL-NEXT: s_mov_b32 s3, 0
+; GFX11-GISEL-NEXT: .p2align 6
; GFX11-GISEL-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -2047,7 +2285,13 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_16(ptr
; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, v0
; GFX11-GISEL-NEXT: s_or_b32 s3, vcc_lo, s3
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s3
+; GFX11-GISEL-NEXT: s_and_not1_b32 s4, s4, exec_lo
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s5
+; GFX11-GISEL-NEXT: s_or_b32 s4, s4, s6
; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB11_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2056,11 +2300,12 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_16(ptr
; GFX12-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn_16:
; GFX12-GISEL: ; %bb.0:
; GFX12-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr4
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr5
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_load_b32 s3, s[0:1], 0x800
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
; GFX12-GISEL-NEXT: s_mov_b32 s3, 0
; GFX12-GISEL-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -2076,7 +2321,14 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_16(ptr
; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX12-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s6, exec_lo, s3
+; GFX12-GISEL-NEXT: s_and_not1_b32 s4, s4, exec_lo
+; GFX12-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: s_and_b32 s6, exec_lo, s5
+; GFX12-GISEL-NEXT: s_or_b32 s4, s4, s6
; GFX12-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB11_1
; GFX12-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2209,22 +2461,34 @@ define i8 @global_atomic_usub_sat_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: global_load_dword v3, v[0:1], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-GISEL-NEXT: s_movk_i32 s6, 0xff00
+; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0xff
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 0xffffff00
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
; GFX9-GISEL-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
-; GFX9-GISEL-NEXT: v_sub_u16_sdwa v3, v4, v2 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, s6, v3
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v7, v3
+; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v3, 8, v7
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v3, v3, v2 clamp
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_or_b32 v6, v7, v5, v3
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB12_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -2232,27 +2496,37 @@ define i8 @global_atomic_usub_sat_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: global_load_dword v3, v[0:1], off
-; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b16 v2, 8, v2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, 0xff
; GFX10-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX10-GISEL-NEXT: .p2align 6
; GFX10-GISEL-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v4
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-GISEL-NEXT: v_lshlrev_b16 v3, 8, v6
; GFX10-GISEL-NEXT: v_sub_nc_u16 v3, v3, v2 clamp
-; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-GISEL-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_and_or_b32 v5, 0xffffff00, v6, v3
; GFX10-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: buffer_gl1_inv
; GFX10-GISEL-NEXT: buffer_gl0_inv
-; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-GISEL-NEXT: s_andn2_b32 s6, s6, exec_lo
+; GFX10-GISEL-NEXT: s_and_b32 s7, exec_lo, s4
+; GFX10-GISEL-NEXT: s_andn2_b32 s5, s5, exec_lo
+; GFX10-GISEL-NEXT: s_or_b32 s6, s6, s7
+; GFX10-GISEL-NEXT: s_and_b32 s7, exec_lo, s6
+; GFX10-GISEL-NEXT: s_or_b32 s5, s5, s7
; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB12_1
; GFX10-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -2260,17 +2534,22 @@ define i8 @global_atomic_usub_sat_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX11-GISEL: ; %bb.0:
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: global_load_b32 v3, v[0:1], off
-; GFX11-GISEL-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX11-GISEL-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
; GFX11-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr2
+; GFX11-GISEL-NEXT: .p2align 6
; GFX11-GISEL-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX11-GISEL-NEXT: v_mov_b32_e32 v4, v3
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_and_b16 v2.h, 0xff, v4.l
+; GFX11-GISEL-NEXT: v_lshlrev_b16 v2.h, 8, v4.l
; GFX11-GISEL-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-GISEL-NEXT: v_lshrrev_b16 v3.l, 8, v2.h
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX11-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -2279,12 +2558,18 @@ define i8 @global_atomic_usub_sat_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX11-GISEL-NEXT: buffer_gl0_inv
; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX11-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_not1_b32 s2, s2, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s3, exec_lo, s0
+; GFX11-GISEL-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-GISEL-NEXT: s_or_b32 s2, s2, s3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_b32 s3, exec_lo, s2
+; GFX11-GISEL-NEXT: s_or_b32 s1, s1, s3
; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB12_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-GISEL-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: global_atomic_usub_sat_8:
@@ -2295,17 +2580,21 @@ define i8 @global_atomic_usub_sat_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: global_load_b32 v3, v[0:1], off
-; GFX12-GISEL-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX12-GISEL-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
; GFX12-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr2
; GFX12-GISEL-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX12-GISEL-NEXT: v_mov_b32_e32 v4, v3
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_and_b16 v2.h, 0xff, v4.l
+; GFX12-GISEL-NEXT: v_lshlrev_b16 v2.h, 8, v4.l
; GFX12-GISEL-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-GISEL-NEXT: v_lshrrev_b16 v3.l, 8, v2.h
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX12-GISEL-NEXT: s_wait_storecnt 0x0
; GFX12-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -2314,12 +2603,22 @@ define i8 @global_atomic_usub_sat_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX12-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-GISEL-NEXT: s_and_not1_b32 s2, s2, exec_lo
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s3, exec_lo, s0
+; GFX12-GISEL-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 s2, s2, s3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s3, exec_lo, s2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 s1, s1, s3
; GFX12-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB12_1
; GFX12-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: global_atomic_usub_sat_8:
@@ -2449,22 +2748,34 @@ define i8 @global_atomic_usub_sat_offset_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: global_load_dword v3, v[0:1], off offset:1024
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-GISEL-NEXT: s_movk_i32 s6, 0xff00
+; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0xff
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 0xffffff00
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
; GFX9-GISEL-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
-; GFX9-GISEL-NEXT: v_sub_u16_sdwa v3, v4, v2 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, s6, v3
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:1024 glc
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v7, v3
+; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v3, 8, v7
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v3, v3, v2 clamp
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_or_b32 v6, v7, v5, v3
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off offset:1024 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB13_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -2472,27 +2783,37 @@ define i8 @global_atomic_usub_sat_offset_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: global_load_dword v3, v[0:1], off offset:1024
-; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b16 v2, 8, v2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, 0xff
; GFX10-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX10-GISEL-NEXT: .p2align 6
; GFX10-GISEL-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v4
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-GISEL-NEXT: v_lshlrev_b16 v3, 8, v6
; GFX10-GISEL-NEXT: v_sub_nc_u16 v3, v3, v2 clamp
-; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-GISEL-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_and_or_b32 v5, 0xffffff00, v6, v3
; GFX10-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:1024 glc
+; GFX10-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:1024 glc
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: buffer_gl1_inv
; GFX10-GISEL-NEXT: buffer_gl0_inv
-; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-GISEL-NEXT: s_andn2_b32 s6, s6, exec_lo
+; GFX10-GISEL-NEXT: s_and_b32 s7, exec_lo, s4
+; GFX10-GISEL-NEXT: s_andn2_b32 s5, s5, exec_lo
+; GFX10-GISEL-NEXT: s_or_b32 s6, s6, s7
+; GFX10-GISEL-NEXT: s_and_b32 s7, exec_lo, s6
+; GFX10-GISEL-NEXT: s_or_b32 s5, s5, s7
; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB13_1
; GFX10-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -2500,17 +2821,22 @@ define i8 @global_atomic_usub_sat_offset_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX11-GISEL: ; %bb.0:
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: global_load_b32 v3, v[0:1], off offset:1024
-; GFX11-GISEL-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX11-GISEL-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
; GFX11-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr2
+; GFX11-GISEL-NEXT: .p2align 6
; GFX11-GISEL-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX11-GISEL-NEXT: v_mov_b32_e32 v4, v3
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_and_b16 v2.h, 0xff, v4.l
+; GFX11-GISEL-NEXT: v_lshlrev_b16 v2.h, 8, v4.l
; GFX11-GISEL-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-GISEL-NEXT: v_lshrrev_b16 v3.l, 8, v2.h
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX11-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:1024 glc
@@ -2519,12 +2845,18 @@ define i8 @global_atomic_usub_sat_offset_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX11-GISEL-NEXT: buffer_gl0_inv
; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX11-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_not1_b32 s2, s2, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s3, exec_lo, s0
+; GFX11-GISEL-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-GISEL-NEXT: s_or_b32 s2, s2, s3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_b32 s3, exec_lo, s2
+; GFX11-GISEL-NEXT: s_or_b32 s1, s1, s3
; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB13_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-GISEL-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: global_atomic_usub_sat_offset_8:
@@ -2535,17 +2867,21 @@ define i8 @global_atomic_usub_sat_offset_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: global_load_b32 v3, v[0:1], off offset:1024
-; GFX12-GISEL-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX12-GISEL-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
; GFX12-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr2
; GFX12-GISEL-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX12-GISEL-NEXT: v_mov_b32_e32 v4, v3
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_and_b16 v2.h, 0xff, v4.l
+; GFX12-GISEL-NEXT: v_lshlrev_b16 v2.h, 8, v4.l
; GFX12-GISEL-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-GISEL-NEXT: v_lshrrev_b16 v3.l, 8, v2.h
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX12-GISEL-NEXT: s_wait_storecnt 0x0
; GFX12-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -2554,12 +2890,22 @@ define i8 @global_atomic_usub_sat_offset_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX12-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-GISEL-NEXT: s_and_not1_b32 s2, s2, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s3, exec_lo, s0
+; GFX12-GISEL-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 s2, s2, s3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s3, exec_lo, s2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 s1, s1, s3
; GFX12-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB13_1
; GFX12-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: global_atomic_usub_sat_offset_8:
@@ -2688,38 +3034,54 @@ define void @global_atomic_usub_sat_nortn_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX9-GISEL-LABEL: global_atomic_usub_sat_nortn_8:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dword v4, v[0:1], off
+; GFX9-GISEL-NEXT: global_load_dword v3, v[0:1], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-GISEL-NEXT: s_movk_i32 s6, 0xff00
+; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v4, 8, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 0xff
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v6, 0xffffff00
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
; GFX9-GISEL-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_sub_u16_sdwa v3, v4, v2 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, s6, v3
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v2, 8, v3
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v2, v2, v4 clamp
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_or_b32 v2, v3, v6, v2
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX9-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB14_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: global_atomic_usub_sat_nortn_8:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: global_load_dword v3, v[0:1], off
-; GFX10-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b16 v4, 8, v2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, 0xff
; GFX10-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX10-GISEL-NEXT: .p2align 6
; GFX10-GISEL-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GFX10-GISEL-NEXT: v_lshlrev_b16 v2, 8, v3
; GFX10-GISEL-NEXT: v_sub_nc_u16 v2, v2, v4 clamp
-; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
; GFX10-GISEL-NEXT: v_and_or_b32 v2, 0xffffff00, v3, v2
; GFX10-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-GISEL-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
@@ -2729,26 +3091,36 @@ define void @global_atomic_usub_sat_nortn_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, v2
; GFX10-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-GISEL-NEXT: s_andn2_b32 s6, s6, exec_lo
+; GFX10-GISEL-NEXT: s_and_b32 s7, exec_lo, s4
+; GFX10-GISEL-NEXT: s_andn2_b32 s5, s5, exec_lo
+; GFX10-GISEL-NEXT: s_or_b32 s6, s6, s7
+; GFX10-GISEL-NEXT: s_and_b32 s7, exec_lo, s6
+; GFX10-GISEL-NEXT: s_or_b32 s5, s5, s7
; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB14_1
; GFX10-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: global_atomic_usub_sat_nortn_8:
; GFX11-GISEL: ; %bb.0:
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-GISEL-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX11-GISEL-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
; GFX11-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr2
+; GFX11-GISEL-NEXT: .p2align 6
; GFX11-GISEL-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-NEXT: v_and_b16 v2.h, 0xff, v4.l
+; GFX11-GISEL-NEXT: v_lshlrev_b16 v2.h, 8, v4.l
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
-; GFX11-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_lshrrev_b16 v3.l, 8, v2.h
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX11-GISEL-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX11-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -2758,11 +3130,17 @@ define void @global_atomic_usub_sat_nortn_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX11-GISEL-NEXT: v_mov_b32_e32 v4, v3
; GFX11-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_not1_b32 s2, s2, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s3, exec_lo, s0
+; GFX11-GISEL-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-GISEL-NEXT: s_or_b32 s2, s2, s3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_b32 s3, exec_lo, s2
+; GFX11-GISEL-NEXT: s_or_b32 s1, s1, s3
; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB14_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: global_atomic_usub_sat_nortn_8:
@@ -2773,16 +3151,19 @@ define void @global_atomic_usub_sat_nortn_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-GISEL-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX12-GISEL-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
; GFX12-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr2
; GFX12-GISEL-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: v_and_b16 v2.h, 0xff, v4.l
+; GFX12-GISEL-NEXT: v_lshlrev_b16 v2.h, 8, v4.l
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
-; GFX12-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_lshrrev_b16 v3.l, 8, v2.h
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX12-GISEL-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX12-GISEL-NEXT: s_wait_storecnt 0x0
; GFX12-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -2792,11 +3173,21 @@ define void @global_atomic_usub_sat_nortn_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX12-GISEL-NEXT: v_mov_b32_e32 v4, v3
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-GISEL-NEXT: s_and_not1_b32 s2, s2, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s3, exec_lo, s0
+; GFX12-GISEL-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 s2, s2, s3
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s3, exec_lo, s2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 s1, s1, s3
; GFX12-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB14_1
; GFX12-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: global_atomic_usub_sat_nortn_8:
@@ -2920,38 +3311,54 @@ define void @global_atomic_usub_sat_offset_nortn_8(ptr addrspace(1) %ptr, i8 %da
; GFX9-GISEL-LABEL: global_atomic_usub_sat_offset_nortn_8:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dword v4, v[0:1], off offset:1024
+; GFX9-GISEL-NEXT: global_load_dword v3, v[0:1], off offset:1024
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-GISEL-NEXT: s_movk_i32 s6, 0xff00
+; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v4, 8, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 0xff
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v6, 0xffffff00
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
; GFX9-GISEL-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_sub_u16_sdwa v3, v4, v2 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, s6, v3
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:1024 glc
+; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v2, 8, v3
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v2, v2, v4 clamp
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_or_b32 v2, v3, v6, v2
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:1024 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX9-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB15_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: global_atomic_usub_sat_offset_nortn_8:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: global_load_dword v3, v[0:1], off offset:1024
-; GFX10-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b16 v4, 8, v2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, 0xff
; GFX10-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX10-GISEL-NEXT: .p2align 6
; GFX10-GISEL-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GFX10-GISEL-NEXT: v_lshlrev_b16 v2, 8, v3
; GFX10-GISEL-NEXT: v_sub_nc_u16 v2, v2, v4 clamp
-; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
; GFX10-GISEL-NEXT: v_and_or_b32 v2, 0xffffff00, v3, v2
; GFX10-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-GISEL-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:1024 glc
@@ -2961,26 +3368,36 @@ define void @global_atomic_usub_sat_offset_nortn_8(ptr addrspace(1) %ptr, i8 %da
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, v2
; GFX10-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-GISEL-NEXT: s_andn2_b32 s6, s6, exec_lo
+; GFX10-GISEL-NEXT: s_and_b32 s7, exec_lo, s4
+; GFX10-GISEL-NEXT: s_andn2_b32 s5, s5, exec_lo
+; GFX10-GISEL-NEXT: s_or_b32 s6, s6, s7
+; GFX10-GISEL-NEXT: s_and_b32 s7, exec_lo, s6
+; GFX10-GISEL-NEXT: s_or_b32 s5, s5, s7
; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB15_1
; GFX10-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: global_atomic_usub_sat_offset_nortn_8:
; GFX11-GISEL: ; %bb.0:
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: global_load_b32 v4, v[0:1], off offset:1024
-; GFX11-GISEL-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX11-GISEL-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
; GFX11-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr2
+; GFX11-GISEL-NEXT: .p2align 6
; GFX11-GISEL-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-NEXT: v_and_b16 v2.h, 0xff, v4.l
+; GFX11-GISEL-NEXT: v_lshlrev_b16 v2.h, 8, v4.l
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
-; GFX11-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_lshrrev_b16 v3.l, 8, v2.h
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX11-GISEL-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX11-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:1024 glc
@@ -2990,11 +3407,17 @@ define void @global_atomic_usub_sat_offset_nortn_8(ptr addrspace(1) %ptr, i8 %da
; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX11-GISEL-NEXT: v_mov_b32_e32 v4, v3
; GFX11-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_not1_b32 s2, s2, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s3, exec_lo, s0
+; GFX11-GISEL-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-GISEL-NEXT: s_or_b32 s2, s2, s3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_b32 s3, exec_lo, s2
+; GFX11-GISEL-NEXT: s_or_b32 s1, s1, s3
; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: global_atomic_usub_sat_offset_nortn_8:
@@ -3005,16 +3428,19 @@ define void @global_atomic_usub_sat_offset_nortn_8(ptr addrspace(1) %ptr, i8 %da
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: global_load_b32 v4, v[0:1], off offset:1024
-; GFX12-GISEL-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX12-GISEL-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
; GFX12-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr2
; GFX12-GISEL-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: v_and_b16 v2.h, 0xff, v4.l
+; GFX12-GISEL-NEXT: v_lshlrev_b16 v2.h, 8, v4.l
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
-; GFX12-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_lshrrev_b16 v3.l, 8, v2.h
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX12-GISEL-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX12-GISEL-NEXT: s_wait_storecnt 0x0
; GFX12-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -3024,11 +3450,21 @@ define void @global_atomic_usub_sat_offset_nortn_8(ptr addrspace(1) %ptr, i8 %da
; GFX12-GISEL-NEXT: v_mov_b32_e32 v4, v3
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-GISEL-NEXT: s_and_not1_b32 s2, s2, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s3, exec_lo, s0
+; GFX12-GISEL-NEXT: s_and_not1_b32 s1, s1, exec_lo
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 s2, s2, s3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s3, exec_lo, s2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 s1, s1, s3
; GFX12-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB15_1
; GFX12-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: global_atomic_usub_sat_offset_nortn_8:
@@ -3153,33 +3589,44 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
; GFX9-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_8:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX9-GISEL-NEXT: s_load_dword s5, s[8:9], 0x8
+; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-GISEL-NEXT: s_movk_i32 s4, 0xff00
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0xff
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xffffff00
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_load_dword s6, s[0:1], 0x400
-; GFX9-GISEL-NEXT: s_and_b32 s5, s5, 0xff
+; GFX9-GISEL-NEXT: s_load_dword s11, s[0:1], 0x400
+; GFX9-GISEL-NEXT: s_lshl_b32 s10, s4, 8
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr4_sgpr5
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, s11
; GFX9-GISEL-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-GISEL-NEXT: v_sub_u16_sdwa v0, v3, s5 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-GISEL-NEXT: v_and_or_b32 v2, v3, s4, v0
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v1, v[2:3], s[0:1] offset:1024 glc
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v3, 8, v4
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v3, v3, s10 clamp
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, v1, v3
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v2, v[3:4], s[0:1] offset:1024 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[12:13], exec, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[12:13]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[12:13], exec, s[6:7]
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[12:13]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB16_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_store_byte v1, v0, s[0:1]
+; GFX9-GISEL-NEXT: global_store_byte v0, v3, s[0:1]
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_8:
@@ -3188,33 +3635,43 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX10-GISEL-NEXT: s_load_dword s3, s[8:9], 0x8
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr4
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr5
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: s_load_dword s2, s[0:1], 0x400
-; GFX10-GISEL-NEXT: s_and_b32 s3, s3, 0xff
+; GFX10-GISEL-NEXT: s_lshl_b32 s3, s3, 8
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX10-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX10-GISEL-NEXT: .p2align 6
; GFX10-GISEL-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, v1
-; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
-; GFX10-GISEL-NEXT: v_sub_nc_u16 v1, v1, s3 clamp
-; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-GISEL-NEXT: v_and_or_b32 v1, 0xffffff00, v2, v1
-; GFX10-GISEL-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] offset:1024 glc
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b16 v2, 8, v3
+; GFX10-GISEL-NEXT: v_sub_nc_u16 v2, v2, s3 clamp
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_and_or_b32 v2, 0xffffff00, v3, v2
+; GFX10-GISEL-NEXT: global_atomic_cmpswap v2, v0, v[2:3], s[0:1] offset:1024 glc
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: buffer_gl1_inv
; GFX10-GISEL-NEXT: buffer_gl0_inv
-; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
; GFX10-GISEL-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX10-GISEL-NEXT: s_andn2_b32 s5, s5, exec_lo
+; GFX10-GISEL-NEXT: s_and_b32 s6, exec_lo, s2
+; GFX10-GISEL-NEXT: s_andn2_b32 s4, s4, exec_lo
+; GFX10-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX10-GISEL-NEXT: s_and_b32 s6, exec_lo, s5
+; GFX10-GISEL-NEXT: s_or_b32 s4, s4, s6
; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB16_1
; GFX10-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_store_byte v0, v1, s[0:1]
+; GFX10-GISEL-NEXT: global_store_byte v0, v2, s[0:1]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_8:
@@ -3222,22 +3679,26 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
; GFX11-GISEL-NEXT: s_clause 0x1
; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-GISEL-NEXT: s_load_b32 s3, s[4:5], 0x8
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr7
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: s_load_b32 s2, s[0:1], 0x400
-; GFX11-GISEL-NEXT: s_and_b32 s3, s3, 0xff
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-GISEL-NEXT: s_lshl_b32 s3, s3, 8
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX11-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX11-GISEL-NEXT: .p2align 6
; GFX11-GISEL-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, v1
-; GFX11-GISEL-NEXT: v_and_b16 v1.l, 0xff, v2.l
+; GFX11-GISEL-NEXT: v_lshlrev_b16 v1.l, 8, v2.l
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_sub_nc_u16 v1.l, v1.l, s3 clamp
-; GFX11-GISEL-NEXT: v_cvt_u32_u16_e32 v1, v1.l
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_lshrrev_b16 v1.l, 8, v1.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX11-GISEL-NEXT: v_and_or_b32 v1, 0xffffff00, v2, v1
; GFX11-GISEL-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:1024 glc
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -3245,11 +3706,17 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
; GFX11-GISEL-NEXT: buffer_gl0_inv
; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX11-GISEL-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_not1_b32 s7, s7, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s8, exec_lo, s2
+; GFX11-GISEL-NEXT: s_and_not1_b32 s6, s6, exec_lo
+; GFX11-GISEL-NEXT: s_or_b32 s7, s7, s8
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_b32 s8, exec_lo, s7
+; GFX11-GISEL-NEXT: s_or_b32 s6, s6, s8
; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB16_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s6
; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x10
; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -3259,10 +3726,12 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
; GFX12-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_8:
; GFX12-GISEL: ; %bb.0:
; GFX12-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr7
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_load_b32 s3, s[0:1], 0x400
-; GFX12-GISEL-NEXT: s_and_b32 s2, s2, 0xff
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, s2, 8
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX12-GISEL-NEXT: s_mov_b32 s3, 0
@@ -3270,11 +3739,12 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, v1
-; GFX12-GISEL-NEXT: v_and_b16 v1.l, 0xff, v2.l
+; GFX12-GISEL-NEXT: v_lshlrev_b16 v1.l, 8, v2.l
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_sub_nc_u16 v1.l, v1.l, s2 clamp
-; GFX12-GISEL-NEXT: v_cvt_u32_u16_e32 v1, v1.l
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_lshrrev_b16 v1.l, 8, v1.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX12-GISEL-NEXT: v_and_or_b32 v1, 0xffffff00, v2, v1
; GFX12-GISEL-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
@@ -3282,11 +3752,18 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
; GFX12-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX12-GISEL-NEXT: s_and_not1_b32 s7, s7, exec_lo
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s8, exec_lo, s3
+; GFX12-GISEL-NEXT: s_and_not1_b32 s6, s6, exec_lo
+; GFX12-GISEL-NEXT: s_or_b32 s7, s7, s8
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: s_and_b32 s8, exec_lo, s7
+; GFX12-GISEL-NEXT: s_or_b32 s6, s6, s8
; GFX12-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB16_1
; GFX12-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s6
; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x10
; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
@@ -3445,25 +3922,36 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_8(ptr a
; GFX9-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn_8:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX9-GISEL-NEXT: s_load_dword s5, s[8:9], 0x8
+; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-GISEL-NEXT: s_movk_i32 s4, 0xff00
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0xff
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0xffffff00
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_load_dword s6, s[0:1], 0x400
-; GFX9-GISEL-NEXT: s_and_b32 s5, s5, 0xff
+; GFX9-GISEL-NEXT: s_load_dword s9, s[0:1], 0x400
+; GFX9-GISEL-NEXT: s_lshl_b32 s8, s4, 8
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr4_sgpr5
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s9
; GFX9-GISEL-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-GISEL-NEXT: v_sub_u16_sdwa v0, v1, s5 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-GISEL-NEXT: v_and_or_b32 v0, v1, s4, v0
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:1024 glc
+; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v0, 8, v1
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v0, v0, s8 clamp
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v1, v3, v0
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v4, v[0:1], s[0:1] offset:1024 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[6:7]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[10:11]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB17_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3475,17 +3963,21 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_8(ptr a
; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX10-GISEL-NEXT: s_load_dword s3, s[8:9], 0x8
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0xff
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr4
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr5
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: s_load_dword s2, s[0:1], 0x400
-; GFX10-GISEL-NEXT: s_and_b32 s3, s3, 0xff
+; GFX10-GISEL-NEXT: s_lshl_b32 s3, s3, 8
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX10-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX10-GISEL-NEXT: .p2align 6
; GFX10-GISEL-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v1
+; GFX10-GISEL-NEXT: v_lshlrev_b16 v0, 8, v1
; GFX10-GISEL-NEXT: v_sub_nc_u16 v0, v0, s3 clamp
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
; GFX10-GISEL-NEXT: v_and_or_b32 v0, 0xffffff00, v1, v0
; GFX10-GISEL-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:1024 glc
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -3494,6 +3986,12 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_8(ptr a
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, v0
; GFX10-GISEL-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX10-GISEL-NEXT: s_andn2_b32 s5, s5, exec_lo
+; GFX10-GISEL-NEXT: s_and_b32 s6, exec_lo, s2
+; GFX10-GISEL-NEXT: s_andn2_b32 s4, s4, exec_lo
+; GFX10-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX10-GISEL-NEXT: s_and_b32 s6, exec_lo, s5
+; GFX10-GISEL-NEXT: s_or_b32 s4, s4, s6
; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB17_1
; GFX10-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3504,20 +4002,25 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_8(ptr a
; GFX11-GISEL-NEXT: s_clause 0x1
; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-GISEL-NEXT: s_load_b32 s3, s[4:5], 0x8
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr4
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: s_load_b32 s2, s[0:1], 0x400
-; GFX11-GISEL-NEXT: s_and_b32 s3, s3, 0xff
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_lshl_b32 s3, s3, 8
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX11-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX11-GISEL-NEXT: .p2align 6
; GFX11-GISEL-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_and_b16 v0.l, 0xff, v1.l
+; GFX11-GISEL-NEXT: v_lshlrev_b16 v0.l, 8, v1.l
; GFX11-GISEL-NEXT: v_sub_nc_u16 v0.l, v0.l, s3 clamp
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX11-GISEL-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_and_or_b32 v0, 0xffffff00, v1, v0
; GFX11-GISEL-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:1024 glc
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -3526,7 +4029,13 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_8(ptr a
; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, v0
; GFX11-GISEL-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s2
+; GFX11-GISEL-NEXT: s_and_not1_b32 s4, s4, exec_lo
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s5
+; GFX11-GISEL-NEXT: s_or_b32 s4, s4, s6
; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB17_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3535,20 +4044,24 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_8(ptr a
; GFX12-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn_8:
; GFX12-GISEL: ; %bb.0:
; GFX12-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr4
+; GFX12-GISEL-NEXT: ; implicit-def: $sgpr5
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_load_b32 s3, s[0:1], 0x400
-; GFX12-GISEL-NEXT: s_and_b32 s2, s2, 0xff
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, s2, 8
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX12-GISEL-NEXT: s_mov_b32 s3, 0
; GFX12-GISEL-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_and_b16 v0.l, 0xff, v1.l
+; GFX12-GISEL-NEXT: v_lshlrev_b16 v0.l, 8, v1.l
; GFX12-GISEL-NEXT: v_sub_nc_u16 v0.l, v0.l, s2 clamp
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX12-GISEL-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_and_or_b32 v0, 0xffffff00, v1, v0
; GFX12-GISEL-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
@@ -3557,7 +4070,14 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_8(ptr a
; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX12-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s6, exec_lo, s3
+; GFX12-GISEL-NEXT: s_and_not1_b32 s4, s4, exec_lo
+; GFX12-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: s_and_b32 s6, exec_lo, s5
+; GFX12-GISEL-NEXT: s_or_b32 s4, s4, s6
; GFX12-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB17_1
; GFX12-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3694,6 +4214,8 @@ define i32 @global_atomic_usub_sat__amdgpu_no_remote_memory(ptr addrspace(1) %pt
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: global_load_dword v3, v[0:1], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
; GFX9-GISEL-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -3704,10 +4226,16 @@ define i32 @global_atomic_usub_sat__amdgpu_no_remote_memory(ptr addrspace(1) %pt
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB18_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -3808,6 +4336,8 @@ define i32 @global_atomic_usub_sat__amdgpu_no_fine_grained_memory(ptr addrspace(
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: global_load_dword v3, v[0:1], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
; GFX9-GISEL-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -3818,10 +4348,16 @@ define i32 @global_atomic_usub_sat__amdgpu_no_fine_grained_memory(ptr addrspace(
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB19_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -3922,6 +4458,8 @@ define i32 @global_atomic_usub_sat__amdgpu_no_fine_grained_memory__amdgpu_no_rem
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: global_load_dword v3, v[0:1], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
; GFX9-GISEL-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -3932,10 +4470,16 @@ define i32 @global_atomic_usub_sat__amdgpu_no_fine_grained_memory__amdgpu_no_rem
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX9-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB20_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index 45e3cee156901..ecb182853f540 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -459,9 +459,8 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s9, exec_lo
-; GFX10-NEXT: s_mov_b32 s8, 0
-; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
@@ -471,11 +470,12 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_dword v8, v4, s[4:7], 0 offen offset:1024
-; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB2_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
+; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: .LBB2_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB2_4 Depth 2
@@ -2405,23 +2405,24 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
-; GFX11-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_mov_b32 s6, s5
+; GFX11-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_mov_b32 s5, s4
; GFX11-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_readfirstlane_b32 s0, v9
; GFX11-NEXT: v_readfirstlane_b32 s1, v10
; GFX11-NEXT: v_readfirstlane_b32 s2, v7
; GFX11-NEXT: v_readfirstlane_b32 s3, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[0:3], 0 offen offset:2048
-; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-NEXT: s_mov_b32 exec_lo, s4
+; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB10_3: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Loop Header: Depth=1
@@ -2469,9 +2470,8 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX10-NEXT: v_mov_b32_e32 v7, v2
; GFX10-NEXT: v_mov_b32_e32 v10, v1
; GFX10-NEXT: v_mov_b32_e32 v9, v0
-; GFX10-NEXT: s_mov_b32 s9, exec_lo
-; GFX10-NEXT: s_mov_b32 s8, 0
-; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v9
; GFX10-NEXT: v_readfirstlane_b32 s5, v10
@@ -2481,11 +2481,12 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[9:10]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[7:8]
; GFX10-NEXT: buffer_load_dwordx2 v[13:14], v4, s[4:7], 0 offen offset:2048
-; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB10_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
+; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: .LBB10_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB10_4 Depth 2
@@ -4411,15 +4412,15 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v6
; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, -4, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v7
; GFX11-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -4430,10 +4431,11 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v10, s[0:3], 0 offen
-; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
@@ -4484,15 +4486,15 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 3, v6
; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, -4, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_not_b32_e32 v11, v7
; GFX11-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -4503,10 +4505,11 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v10, s[0:3], 0 offen
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
@@ -4557,9 +4560,8 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
-; GFX10-NEXT: s_mov_b32 s9, exec_lo
-; GFX10-NEXT: s_mov_b32 s8, 0
-; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: v_and_b32_e32 v4, 3, v6
; GFX10-NEXT: v_and_b32_e32 v10, -4, v6
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
@@ -4574,11 +4576,12 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_dword v7, v10, s[4:7], 0 offen
-; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB15_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
+; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB15_4 Depth 2
@@ -6249,15 +6252,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-TRUE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -6268,11 +6271,12 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
-; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB18_3: ; %atomicrmw.start
@@ -6334,15 +6338,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-FAKE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -6353,11 +6357,12 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB18_3: ; %atomicrmw.start
@@ -6418,9 +6423,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX10-NEXT: s_mov_b32 s9, exec_lo
-; GFX10-NEXT: s_mov_b32 s8, 0
-; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: v_and_b32_e32 v6, 3, v4
; GFX10-NEXT: v_and_b32_e32 v8, -4, v4
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
@@ -6435,12 +6439,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_dword v6, v8, s[4:7], 0 offen
-; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB18_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB18_4 Depth 2
@@ -7477,9 +7482,9 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX11-TRUE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -7489,13 +7494,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_u16 v8, v4, s[0:3], 0 offen offset:1024
-; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX11-TRUE16-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -7505,12 +7510,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX11-TRUE16-NEXT: ; %bb.4:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v6.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
@@ -7553,9 +7559,9 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX11-FAKE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -7565,13 +7571,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX11-FAKE16-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -7581,12 +7587,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1026
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX11-FAKE16-NEXT: ; %bb.4:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
@@ -7629,9 +7636,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s9, exec_lo
-; GFX10-NEXT: s_mov_b32 s8, 0
-; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
@@ -7641,13 +7647,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_ushort v6, v4, s[4:7], 0 offen offset:1024
-; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB21_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 exec_lo, s9
-; GFX10-NEXT: s_mov_b32 s9, exec_lo
-; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
@@ -7657,13 +7663,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_ushort v7, v4, s[4:7], 0 offen offset:1026
-; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB21_3
; GFX10-NEXT: ; %bb.4:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
+; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB21_6 Depth 2
@@ -10358,9 +10365,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX11-TRUE16-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -10370,13 +10377,13 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1024
-; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX11-TRUE16-NEXT: .LBB28_3: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -10386,28 +10393,29 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_3
; GFX11-TRUE16-NEXT: ; %bb.4:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB28_5: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-TRUE16-NEXT: ; Child Loop BB28_6 Depth 2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, v6, v8 :: v_dual_add_f32 v5, v5, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v6, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
@@ -10454,9 +10462,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX11-FAKE16-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -10466,13 +10474,13 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX11-FAKE16-NEXT: .LBB28_3: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -10482,28 +10490,29 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1026
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB28_3
; GFX11-FAKE16-NEXT: ; %bb.4:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB28_5: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-FAKE16-NEXT: ; Child Loop BB28_6 Depth 2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_add_f32 v6, v6, v9 :: v_dual_add_f32 v5, v5, v8
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v11, v6, 16, 1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
@@ -10550,9 +10559,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s9, exec_lo
-; GFX10-NEXT: s_mov_b32 s8, 0
-; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
@@ -10562,13 +10570,13 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_ushort v6, v4, s[4:7], 0 offen offset:1024
-; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB28_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 exec_lo, s9
-; GFX10-NEXT: s_mov_b32 s9, exec_lo
-; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB28_3: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
@@ -10578,15 +10586,16 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_ushort v7, v4, s[4:7], 0 offen offset:1026
-; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB28_3
; GFX10-NEXT: ; %bb.4:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
+; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: .LBB28_5: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB28_6 Depth 2
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index 1078b38f52f4c..682f6d088790b 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -1635,24 +1635,25 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
-; GFX11-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_mov_b32 s6, s5
+; GFX11-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_mov_b32 s5, s4
; GFX11-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_readfirstlane_b32 s0, v9
; GFX11-NEXT: v_readfirstlane_b32 s1, v10
; GFX11-NEXT: v_readfirstlane_b32 s2, v7
; GFX11-NEXT: v_readfirstlane_b32 s3, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[0:3], 0 offen offset:2048
-; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: s_cbranch_execnz .LBB7_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-NEXT: s_mov_b32 exec_lo, s4
; GFX11-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB7_3: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Loop Header: Depth=1
@@ -3528,15 +3529,15 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, -4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 3, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v9, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v6
; GFX11-TRUE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -3547,11 +3548,12 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v10, s[0:3], 0 offen
-; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.l, v5.l, v5.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
@@ -3603,15 +3605,15 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-FAKE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -3622,11 +3624,12 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: v_max_f16_e32 v10, v5, v5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
@@ -3678,9 +3681,8 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX10-NEXT: s_mov_b32 s9, exec_lo
-; GFX10-NEXT: s_mov_b32 s8, 0
-; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: v_and_b32_e32 v6, 3, v4
; GFX10-NEXT: v_and_b32_e32 v8, -4, v4
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
@@ -3695,12 +3697,13 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_dword v6, v8, s[4:7], 0 offen
-; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB12_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: v_max_f16_e32 v10, v5, v5
+; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB12_4 Depth 2
@@ -5382,15 +5385,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -5401,11 +5404,12 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
-; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start
@@ -5467,15 +5471,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -5486,11 +5490,12 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB15_3: ; %atomicrmw.start
@@ -5551,9 +5556,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX10-NEXT: s_mov_b32 s9, exec_lo
-; GFX10-NEXT: s_mov_b32 s8, 0
-; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: v_and_b32_e32 v6, 3, v4
; GFX10-NEXT: v_and_b32_e32 v8, -4, v4
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
@@ -5568,12 +5572,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_dword v6, v8, s[4:7], 0 offen
-; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB15_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB15_4 Depth 2
@@ -7063,9 +7068,9 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX11-TRUE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -7075,13 +7080,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1024
-; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX11-TRUE16-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -7091,13 +7096,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_3
; GFX11-TRUE16-NEXT: ; %bb.4:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
; GFX11-TRUE16-NEXT: v_pk_max_f16 v8, v5, v5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
@@ -7142,9 +7148,9 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX11-FAKE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -7154,13 +7160,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX11-FAKE16-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -7170,13 +7176,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1026
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_3
; GFX11-FAKE16-NEXT: ; %bb.4:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
; GFX11-FAKE16-NEXT: v_pk_max_f16 v8, v5, v5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
@@ -7221,9 +7228,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s9, exec_lo
-; GFX10-NEXT: s_mov_b32 s8, 0
-; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
@@ -7233,13 +7239,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_ushort v6, v4, s[4:7], 0 offen offset:1024
-; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB18_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 exec_lo, s9
-; GFX10-NEXT: s_mov_b32 s9, exec_lo
-; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
@@ -7249,14 +7255,15 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_ushort v7, v4, s[4:7], 0 offen offset:1026
-; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB18_3
; GFX10-NEXT: ; %bb.4:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
; GFX10-NEXT: v_pk_max_f16 v8, v5, v5
+; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB18_6 Depth 2
@@ -9155,9 +9162,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX11-TRUE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -9167,13 +9174,13 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1024
-; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX11-TRUE16-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -9183,28 +9190,29 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX11-TRUE16-NEXT: ; %bb.4:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-TRUE16-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_max_f32 v6, v6, v8 :: v_dual_max_f32 v5, v5, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v6, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
@@ -9251,9 +9259,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX11-FAKE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -9263,13 +9271,13 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX11-FAKE16-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -9279,28 +9287,29 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1026
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX11-FAKE16-NEXT: ; %bb.4:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-FAKE16-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_max_f32 v6, v6, v9 :: v_dual_max_f32 v5, v5, v8
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v11, v6, 16, 1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
@@ -9347,9 +9356,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s9, exec_lo
-; GFX10-NEXT: s_mov_b32 s8, 0
-; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
@@ -9359,13 +9367,13 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_ushort v6, v4, s[4:7], 0 offen offset:1024
-; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB21_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 exec_lo, s9
-; GFX10-NEXT: s_mov_b32 s9, exec_lo
-; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
@@ -9375,15 +9383,16 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_ushort v7, v4, s[4:7], 0 offen offset:1026
-; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB21_3
; GFX10-NEXT: ; %bb.4:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
+; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB21_6 Depth 2
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index e5a87c78d54d2..31319c9448031 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -1635,24 +1635,25 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
-; GFX11-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_mov_b32 s6, s5
+; GFX11-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_mov_b32 s5, s4
; GFX11-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_readfirstlane_b32 s0, v9
; GFX11-NEXT: v_readfirstlane_b32 s1, v10
; GFX11-NEXT: v_readfirstlane_b32 s2, v7
; GFX11-NEXT: v_readfirstlane_b32 s3, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[0:3], 0 offen offset:2048
-; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: s_cbranch_execnz .LBB7_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-NEXT: s_mov_b32 exec_lo, s4
; GFX11-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB7_3: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Loop Header: Depth=1
@@ -3528,15 +3529,15 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, -4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 3, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v9, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v6
; GFX11-TRUE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -3547,11 +3548,12 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v10, s[0:3], 0 offen
-; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.l, v5.l, v5.l
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
@@ -3603,15 +3605,15 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-FAKE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -3622,11 +3624,12 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: v_max_f16_e32 v10, v5, v5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
@@ -3678,9 +3681,8 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX10-NEXT: s_mov_b32 s9, exec_lo
-; GFX10-NEXT: s_mov_b32 s8, 0
-; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: v_and_b32_e32 v6, 3, v4
; GFX10-NEXT: v_and_b32_e32 v8, -4, v4
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
@@ -3695,12 +3697,13 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_dword v6, v8, s[4:7], 0 offen
-; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB12_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: v_max_f16_e32 v10, v5, v5
+; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB12_4 Depth 2
@@ -5382,15 +5385,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -5401,11 +5404,12 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
-; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start
@@ -5467,15 +5471,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -5486,11 +5490,12 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB15_3: ; %atomicrmw.start
@@ -5551,9 +5556,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX10-NEXT: s_mov_b32 s9, exec_lo
-; GFX10-NEXT: s_mov_b32 s8, 0
-; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: v_and_b32_e32 v6, 3, v4
; GFX10-NEXT: v_and_b32_e32 v8, -4, v4
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
@@ -5568,12 +5572,13 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_dword v6, v8, s[4:7], 0 offen
-; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB15_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB15_4 Depth 2
@@ -7063,9 +7068,9 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX11-TRUE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -7075,13 +7080,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1024
-; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX11-TRUE16-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -7091,13 +7096,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_3
; GFX11-TRUE16-NEXT: ; %bb.4:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
; GFX11-TRUE16-NEXT: v_pk_max_f16 v8, v5, v5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
@@ -7142,9 +7148,9 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX11-FAKE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -7154,13 +7160,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX11-FAKE16-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -7170,13 +7176,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1026
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_3
; GFX11-FAKE16-NEXT: ; %bb.4:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
; GFX11-FAKE16-NEXT: v_pk_max_f16 v8, v5, v5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
@@ -7221,9 +7228,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s9, exec_lo
-; GFX10-NEXT: s_mov_b32 s8, 0
-; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
@@ -7233,13 +7239,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_ushort v6, v4, s[4:7], 0 offen offset:1024
-; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB18_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 exec_lo, s9
-; GFX10-NEXT: s_mov_b32 s9, exec_lo
-; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
@@ -7249,14 +7255,15 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_ushort v7, v4, s[4:7], 0 offen offset:1026
-; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB18_3
; GFX10-NEXT: ; %bb.4:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
; GFX10-NEXT: v_pk_max_f16 v8, v5, v5
+; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB18_6 Depth 2
@@ -9155,9 +9162,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX11-TRUE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -9167,13 +9174,13 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1024
-; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX11-TRUE16-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -9183,28 +9190,29 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX11-TRUE16-NEXT: ; %bb.4:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-TRUE16-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_min_f32 v6, v6, v8 :: v_dual_min_f32 v5, v5, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v6, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
@@ -9251,9 +9259,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX11-FAKE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -9263,13 +9271,13 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX11-FAKE16-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -9279,28 +9287,29 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1026
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX11-FAKE16-NEXT: ; %bb.4:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-FAKE16-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_min_f32 v6, v6, v9 :: v_dual_min_f32 v5, v5, v8
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v11, v6, 16, 1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
@@ -9347,9 +9356,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s9, exec_lo
-; GFX10-NEXT: s_mov_b32 s8, 0
-; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
@@ -9359,13 +9367,13 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_ushort v6, v4, s[4:7], 0 offen offset:1024
-; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB21_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 exec_lo, s9
-; GFX10-NEXT: s_mov_b32 s9, exec_lo
-; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
@@ -9375,15 +9383,16 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_ushort v7, v4, s[4:7], 0 offen offset:1026
-; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: s_cbranch_execnz .LBB21_3
; GFX10-NEXT: ; %bb.4:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
+; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB21_6 Depth 2
diff --git a/llvm/test/CodeGen/AMDGPU/codegen-prepare-addrspacecast-non-null.ll b/llvm/test/CodeGen/AMDGPU/codegen-prepare-addrspacecast-non-null.ll
index 70c2975dd2988..7aa8180b6fb7d 100644
--- a/llvm/test/CodeGen/AMDGPU/codegen-prepare-addrspacecast-non-null.ll
+++ b/llvm/test/CodeGen/AMDGPU/codegen-prepare-addrspacecast-non-null.ll
@@ -318,34 +318,73 @@ define void @recursive_phis(i1 %cond, ptr addrspace(5) %ptr) {
; OPT: [[END]]:
; OPT-NEXT: ret void
;
-; ASM-LABEL: recursive_phis:
-; ASM: ; %bb.0: ; %entry
-; ASM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; ASM-NEXT: v_and_b32_e32 v0, 1, v0
-; ASM-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; ASM-NEXT: v_lshrrev_b32_e64 v0, 6, s32
-; ASM-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; ASM-NEXT: ; %bb.1: ; %then
-; ASM-NEXT: v_and_b32_e32 v0, 0xffff, v1
-; ASM-NEXT: ; %bb.2: ; %finallyendcf.split
-; ASM-NEXT: s_or_b64 exec, exec, s[4:5]
-; ASM-NEXT: s_mov_b64 s[4:5], src_private_base
-; ASM-NEXT: s_xor_b64 s[8:9], vcc, -1
-; ASM-NEXT: s_mov_b64 s[6:7], 0
-; ASM-NEXT: v_mov_b32_e32 v2, 7
-; ASM-NEXT: .LBB11_3: ; %finally
-; ASM-NEXT: ; =>This Inner Loop Header: Depth=1
-; ASM-NEXT: s_and_b64 s[10:11], exec, s[8:9]
-; ASM-NEXT: s_or_b64 s[6:7], s[10:11], s[6:7]
-; ASM-NEXT: v_mov_b32_e32 v1, s5
-; ASM-NEXT: flat_store_dword v[0:1], v2
-; ASM-NEXT: s_waitcnt vmcnt(0)
-; ASM-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; ASM-NEXT: s_cbranch_execnz .LBB11_3
-; ASM-NEXT: ; %bb.4: ; %end
-; ASM-NEXT: s_or_b64 exec, exec, s[6:7]
-; ASM-NEXT: s_waitcnt lgkmcnt(0)
-; ASM-NEXT: s_setpc_b64 s[30:31]
+; DAGISEL-ASM-LABEL: recursive_phis:
+; DAGISEL-ASM: ; %bb.0: ; %entry
+; DAGISEL-ASM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; DAGISEL-ASM-NEXT: v_and_b32_e32 v0, 1, v0
+; DAGISEL-ASM-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
+; DAGISEL-ASM-NEXT: v_lshrrev_b32_e64 v0, 6, s32
+; DAGISEL-ASM-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; DAGISEL-ASM-NEXT: ; %bb.1: ; %then
+; DAGISEL-ASM-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; DAGISEL-ASM-NEXT: ; %bb.2: ; %finallyendcf.split
+; DAGISEL-ASM-NEXT: s_or_b64 exec, exec, s[4:5]
+; DAGISEL-ASM-NEXT: s_mov_b64 s[4:5], src_private_base
+; DAGISEL-ASM-NEXT: s_xor_b64 s[8:9], vcc, -1
+; DAGISEL-ASM-NEXT: s_mov_b64 s[6:7], 0
+; DAGISEL-ASM-NEXT: v_mov_b32_e32 v2, 7
+; DAGISEL-ASM-NEXT: .LBB11_3: ; %finally
+; DAGISEL-ASM-NEXT: ; =>This Inner Loop Header: Depth=1
+; DAGISEL-ASM-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; DAGISEL-ASM-NEXT: s_or_b64 s[6:7], s[10:11], s[6:7]
+; DAGISEL-ASM-NEXT: v_mov_b32_e32 v1, s5
+; DAGISEL-ASM-NEXT: flat_store_dword v[0:1], v2
+; DAGISEL-ASM-NEXT: s_waitcnt vmcnt(0)
+; DAGISEL-ASM-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; DAGISEL-ASM-NEXT: s_cbranch_execnz .LBB11_3
+; DAGISEL-ASM-NEXT: ; %bb.4: ; %end
+; DAGISEL-ASM-NEXT: s_or_b64 exec, exec, s[6:7]
+; DAGISEL-ASM-NEXT: s_waitcnt lgkmcnt(0)
+; DAGISEL-ASM-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-ASM-LABEL: recursive_phis:
+; GISEL-ASM: ; %bb.0: ; %entry
+; GISEL-ASM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-ASM-NEXT: v_and_b32_e32 v0, 1, v0
+; GISEL-ASM-NEXT: s_mov_b64 s[4:5], exec
+; GISEL-ASM-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GISEL-ASM-NEXT: s_lshr_b32 s6, s32, 6
+; GISEL-ASM-NEXT: s_xor_b64 s[4:5], vcc, s[4:5]
+; GISEL-ASM-NEXT: v_mov_b32_e32 v0, s6
+; GISEL-ASM-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GISEL-ASM-NEXT: ; %bb.1: ; %then
+; GISEL-ASM-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GISEL-ASM-NEXT: ; %bb.2: ; %finallyendcf.split
+; GISEL-ASM-NEXT: s_or_b64 exec, exec, s[6:7]
+; GISEL-ASM-NEXT: s_mov_b64 s[8:9], src_private_base
+; GISEL-ASM-NEXT: s_mov_b64 s[6:7], 0
+; GISEL-ASM-NEXT: v_mov_b32_e32 v1, s9
+; GISEL-ASM-NEXT: v_mov_b32_e32 v2, 7
+; GISEL-ASM-NEXT: ; implicit-def: $sgpr8_sgpr9
+; GISEL-ASM-NEXT: ; implicit-def: $sgpr10_sgpr11
+; GISEL-ASM-NEXT: .LBB11_3: ; %finally
+; GISEL-ASM-NEXT: ; =>This Inner Loop Header: Depth=1
+; GISEL-ASM-NEXT: s_and_b64 s[12:13], exec, s[4:5]
+; GISEL-ASM-NEXT: s_or_b64 s[6:7], s[12:13], s[6:7]
+; GISEL-ASM-NEXT: s_andn2_b64 s[10:11], s[10:11], exec
+; GISEL-ASM-NEXT: s_and_b64 s[12:13], exec, s[6:7]
+; GISEL-ASM-NEXT: s_or_b64 s[10:11], s[10:11], s[12:13]
+; GISEL-ASM-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GISEL-ASM-NEXT: s_and_b64 s[12:13], exec, s[10:11]
+; GISEL-ASM-NEXT: s_or_b64 s[8:9], s[8:9], s[12:13]
+; GISEL-ASM-NEXT: flat_store_dword v[0:1], v2
+; GISEL-ASM-NEXT: s_waitcnt vmcnt(0)
+; GISEL-ASM-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GISEL-ASM-NEXT: s_cbranch_execnz .LBB11_3
+; GISEL-ASM-NEXT: ; %bb.4: ; %end
+; GISEL-ASM-NEXT: s_or_b64 exec, exec, s[8:9]
+; GISEL-ASM-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-ASM-NEXT: s_setpc_b64 s[30:31]
entry:
%alloca = alloca i8, addrspace(5)
br i1 %cond, label %then, label %else
diff --git a/llvm/test/CodeGen/AMDGPU/div_i128.ll b/llvm/test/CodeGen/AMDGPU/div_i128.ll
index 0b067ebbee0e3..40048779284f3 100644
--- a/llvm/test/CodeGen/AMDGPU/div_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_i128.ll
@@ -100,21 +100,22 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_sub_u32_e32 v2, 63, v2
; GFX9-NEXT: v_or_b32_e32 v5, v3, v5
; GFX9-NEXT: v_lshlrev_b64 v[2:3], v2, v[8:9]
-; GFX9-NEXT: s_xor_b64 s[8:9], vcc, -1
+; GFX9-NEXT: s_xor_b64 s[10:11], vcc, -1
; GFX9-NEXT: v_or_b32_e32 v4, v4, v6
; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v7
; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
; GFX9-NEXT: v_lshlrev_b64 v[4:5], v7, v[8:9]
; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v7
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: v_mov_b32_e32 v7, 0
; GFX9-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
-; GFX9-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GFX9-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GFX9-NEXT: s_and_saveexec_b64 s[4:5], s[10:11]
+; GFX9-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB0_5
; GFX9-NEXT: ; %bb.2: ; %udiv-preheader
; GFX9-NEXT: v_sub_u32_e32 v12, 64, v23
@@ -139,7 +140,6 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_addc_co_u32_e32 v29, vcc, -1, v1, vcc
; GFX9-NEXT: v_mov_b32_e32 v8, 0
; GFX9-NEXT: v_mov_b32_e32 v9, 0
-; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: v_mov_b32_e32 v15, 0
; GFX9-NEXT: v_mov_b32_e32 v16, 0
; GFX9-NEXT: v_mov_b32_e32 v7, 0
@@ -179,15 +179,15 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_or3_b32 v2, v2, v6, v8
; GFX9-NEXT: v_and_b32_e32 v6, 1, v30
; GFX9-NEXT: v_or3_b32 v3, v3, 0, v9
-; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX9-NEXT: v_mov_b32_e32 v16, v7
; GFX9-NEXT: v_mov_b32_e32 v15, v6
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX9-NEXT: s_cbranch_execnz .LBB0_3
; GFX9-NEXT: ; %bb.4: ; %Flow
-; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: .LBB0_5: ; %Flow2
; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX9-NEXT: .LBB0_5: ; %Flow2
+; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX9-NEXT: v_lshlrev_b64 v[12:13], 1, v[2:3]
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 1, v[4:5]
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 31, v5
@@ -950,6 +950,7 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_mov_b64 s[4:5], 0
; GFX9-O0-NEXT: v_writelane_b32 v31, s4, 8
; GFX9-O0-NEXT: v_writelane_b32 v31, s5, 9
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
@@ -1262,21 +1263,22 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-NEXT: v_sub_u32_e32 v2, 63, v2
; GFX9-G-NEXT: v_or_b32_e32 v5, v3, v5
; GFX9-G-NEXT: v_lshlrev_b64 v[2:3], v2, v[8:9]
-; GFX9-G-NEXT: s_xor_b64 s[8:9], vcc, -1
+; GFX9-G-NEXT: s_xor_b64 s[10:11], vcc, -1
; GFX9-G-NEXT: v_or_b32_e32 v4, v4, v6
; GFX9-G-NEXT: v_cmp_gt_u32_e32 vcc, 64, v7
; GFX9-G-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
; GFX9-G-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
; GFX9-G-NEXT: v_lshlrev_b64 v[4:5], v7, v[8:9]
; GFX9-G-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v7
+; GFX9-G-NEXT: s_mov_b64 s[8:9], 0
; GFX9-G-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[4:5]
; GFX9-G-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[4:5]
; GFX9-G-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
; GFX9-G-NEXT: v_mov_b32_e32 v6, 0
; GFX9-G-NEXT: v_mov_b32_e32 v7, 0
; GFX9-G-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
-; GFX9-G-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GFX9-G-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GFX9-G-NEXT: s_and_saveexec_b64 s[4:5], s[10:11]
+; GFX9-G-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
; GFX9-G-NEXT: s_cbranch_execz .LBB0_5
; GFX9-G-NEXT: ; %bb.2: ; %udiv-preheader
; GFX9-G-NEXT: v_sub_u32_e32 v12, 64, v23
@@ -1301,7 +1303,6 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-NEXT: v_addc_co_u32_e32 v29, vcc, -1, v1, vcc
; GFX9-G-NEXT: v_mov_b32_e32 v8, 0
; GFX9-G-NEXT: v_mov_b32_e32 v9, 0
-; GFX9-G-NEXT: s_mov_b64 s[4:5], 0
; GFX9-G-NEXT: v_mov_b32_e32 v15, 0
; GFX9-G-NEXT: v_mov_b32_e32 v16, 0
; GFX9-G-NEXT: v_mov_b32_e32 v7, 0
@@ -1341,15 +1342,15 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-NEXT: v_or3_b32 v2, v2, v6, v8
; GFX9-G-NEXT: v_and_b32_e32 v6, 1, v30
; GFX9-G-NEXT: v_or3_b32 v3, v3, 0, v9
-; GFX9-G-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-G-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX9-G-NEXT: v_mov_b32_e32 v16, v7
; GFX9-G-NEXT: v_mov_b32_e32 v15, v6
-; GFX9-G-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-G-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX9-G-NEXT: s_cbranch_execnz .LBB0_3
; GFX9-G-NEXT: ; %bb.4: ; %Flow
-; GFX9-G-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-G-NEXT: .LBB0_5: ; %Flow2
; GFX9-G-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX9-G-NEXT: .LBB0_5: ; %Flow2
+; GFX9-G-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX9-G-NEXT: v_lshlrev_b64 v[12:13], 1, v[2:3]
; GFX9-G-NEXT: v_lshlrev_b64 v[0:1], 1, v[4:5]
; GFX9-G-NEXT: v_lshrrev_b32_e32 v2, 31, v5
@@ -2112,6 +2113,7 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
; GFX9-G-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s4, 8
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s5, 9
; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
@@ -2409,20 +2411,21 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_addc_co_u32_e32 v21, vcc, 0, v15, vcc
; GFX9-NEXT: v_or_b32_e32 v11, v9, v11
; GFX9-NEXT: v_lshlrev_b64 v[8:9], v8, v[0:1]
-; GFX9-NEXT: s_xor_b64 s[8:9], vcc, -1
+; GFX9-NEXT: s_xor_b64 s[10:11], vcc, -1
; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v13
; GFX9-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc
; GFX9-NEXT: v_lshlrev_b64 v[10:11], v13, v[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v13
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
; GFX9-NEXT: v_cndmask_b32_e64 v9, v9, v3, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e64 v8, v8, v2, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e32 v11, 0, v11, vcc
; GFX9-NEXT: v_mov_b32_e32 v12, 0
; GFX9-NEXT: v_mov_b32_e32 v13, 0
; GFX9-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc
-; GFX9-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GFX9-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GFX9-NEXT: s_and_saveexec_b64 s[4:5], s[10:11]
+; GFX9-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB1_5
; GFX9-NEXT: ; %bb.2: ; %udiv-preheader
; GFX9-NEXT: v_sub_u32_e32 v14, 64, v18
@@ -2447,7 +2450,6 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, -1, v7, vcc
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: v_mov_b32_e32 v16, 0
; GFX9-NEXT: v_mov_b32_e32 v17, 0
; GFX9-NEXT: v_mov_b32_e32 v13, 0
@@ -2487,15 +2489,15 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_or3_b32 v8, v8, v12, v0
; GFX9-NEXT: v_and_b32_e32 v12, 1, v26
; GFX9-NEXT: v_or3_b32 v9, v9, 0, v1
-; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX9-NEXT: v_mov_b32_e32 v17, v13
; GFX9-NEXT: v_mov_b32_e32 v16, v12
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX9-NEXT: s_cbranch_execnz .LBB1_3
; GFX9-NEXT: ; %bb.4: ; %Flow
-; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: .LBB1_5: ; %Flow2
; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX9-NEXT: .LBB1_5: ; %Flow2
+; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX9-NEXT: v_lshlrev_b64 v[8:9], 1, v[8:9]
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 1, v[10:11]
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 31, v11
@@ -3174,6 +3176,7 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_mov_b64 s[4:5], 0
; GFX9-O0-NEXT: v_writelane_b32 v31, s4, 8
; GFX9-O0-NEXT: v_writelane_b32 v31, s5, 9
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
@@ -3435,20 +3438,21 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-NEXT: v_addc_co_u32_e32 v21, vcc, 0, v15, vcc
; GFX9-G-NEXT: v_or_b32_e32 v11, v9, v11
; GFX9-G-NEXT: v_lshlrev_b64 v[8:9], v8, v[0:1]
-; GFX9-G-NEXT: s_xor_b64 s[8:9], vcc, -1
+; GFX9-G-NEXT: s_xor_b64 s[10:11], vcc, -1
; GFX9-G-NEXT: v_cmp_gt_u32_e32 vcc, 64, v13
; GFX9-G-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc
; GFX9-G-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc
; GFX9-G-NEXT: v_lshlrev_b64 v[10:11], v13, v[0:1]
; GFX9-G-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v13
+; GFX9-G-NEXT: s_mov_b64 s[8:9], 0
; GFX9-G-NEXT: v_cndmask_b32_e64 v9, v9, v3, s[4:5]
; GFX9-G-NEXT: v_cndmask_b32_e64 v8, v8, v2, s[4:5]
; GFX9-G-NEXT: v_cndmask_b32_e32 v11, 0, v11, vcc
; GFX9-G-NEXT: v_mov_b32_e32 v12, 0
; GFX9-G-NEXT: v_mov_b32_e32 v13, 0
; GFX9-G-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc
-; GFX9-G-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GFX9-G-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GFX9-G-NEXT: s_and_saveexec_b64 s[4:5], s[10:11]
+; GFX9-G-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
; GFX9-G-NEXT: s_cbranch_execz .LBB1_5
; GFX9-G-NEXT: ; %bb.2: ; %udiv-preheader
; GFX9-G-NEXT: v_sub_u32_e32 v14, 64, v18
@@ -3473,7 +3477,6 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-NEXT: v_addc_co_u32_e32 v25, vcc, -1, v7, vcc
; GFX9-G-NEXT: v_mov_b32_e32 v0, 0
; GFX9-G-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-G-NEXT: s_mov_b64 s[4:5], 0
; GFX9-G-NEXT: v_mov_b32_e32 v16, 0
; GFX9-G-NEXT: v_mov_b32_e32 v17, 0
; GFX9-G-NEXT: v_mov_b32_e32 v13, 0
@@ -3513,15 +3516,15 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-NEXT: v_or3_b32 v8, v8, v12, v0
; GFX9-G-NEXT: v_and_b32_e32 v12, 1, v26
; GFX9-G-NEXT: v_or3_b32 v9, v9, 0, v1
-; GFX9-G-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-G-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX9-G-NEXT: v_mov_b32_e32 v17, v13
; GFX9-G-NEXT: v_mov_b32_e32 v16, v12
-; GFX9-G-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-G-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX9-G-NEXT: s_cbranch_execnz .LBB1_3
; GFX9-G-NEXT: ; %bb.4: ; %Flow
-; GFX9-G-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-G-NEXT: .LBB1_5: ; %Flow2
; GFX9-G-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX9-G-NEXT: .LBB1_5: ; %Flow2
+; GFX9-G-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX9-G-NEXT: v_lshlrev_b64 v[8:9], 1, v[8:9]
; GFX9-G-NEXT: v_lshlrev_b64 v[0:1], 1, v[10:11]
; GFX9-G-NEXT: v_lshrrev_b32_e32 v2, 31, v11
@@ -4200,6 +4203,7 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
; GFX9-G-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s4, 8
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s5, 9
; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
diff --git a/llvm/test/CodeGen/AMDGPU/div_v2i128.ll b/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
index db6dc08e185fc..cd91b72a78b8f 100644
--- a/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
@@ -71,31 +71,31 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[8:9]
; SDAG-NEXT: v_cmp_eq_u64_e64 s[8:9], 0, v[10:11]
-; SDAG-NEXT: v_mov_b32_e32 v26, v24
+; SDAG-NEXT: s_movk_i32 s10, 0x7f
; SDAG-NEXT: v_cndmask_b32_e64 v0, v1, v0, s[8:9]
; SDAG-NEXT: v_and_b32_e32 v0, 1, v0
; SDAG-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v0
; SDAG-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
-; SDAG-NEXT: s_xor_b64 s[10:11], s[4:5], -1
+; SDAG-NEXT: s_xor_b64 s[8:9], s[4:5], -1
; SDAG-NEXT: v_cndmask_b32_e64 v1, v19, 0, s[4:5]
-; SDAG-NEXT: s_movk_i32 s8, 0x7f
; SDAG-NEXT: v_cndmask_b32_e64 v0, v18, 0, s[4:5]
-; SDAG-NEXT: s_and_b64 s[10:11], s[10:11], s[6:7]
+; SDAG-NEXT: s_and_b64 s[8:9], s[8:9], s[6:7]
+; SDAG-NEXT: v_mov_b32_e32 v26, v24
; SDAG-NEXT: v_mov_b32_e32 v27, v25
; SDAG-NEXT: v_cndmask_b32_e64 v20, v17, 0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v21, v16, 0, s[4:5]
-; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[10:11]
+; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[8:9]
; SDAG-NEXT: s_cbranch_execz .LBB0_6
; SDAG-NEXT: ; %bb.1: ; %udiv-bb15
; SDAG-NEXT: v_add_i32_e32 v30, vcc, 1, v8
; SDAG-NEXT: v_addc_u32_e32 v31, vcc, 0, v9, vcc
; SDAG-NEXT: v_addc_u32_e32 v32, vcc, 0, v10, vcc
; SDAG-NEXT: v_addc_u32_e64 v33, s[4:5], 0, v11, vcc
-; SDAG-NEXT: v_sub_i32_e32 v11, vcc, s8, v8
+; SDAG-NEXT: v_sub_i32_e32 v11, vcc, s10, v8
; SDAG-NEXT: v_sub_i32_e32 v9, vcc, 64, v11
; SDAG-NEXT: v_lshl_b64 v[0:1], v[18:19], v11
; SDAG-NEXT: v_lshr_b64 v[9:10], v[16:17], v9
-; SDAG-NEXT: s_xor_b64 s[8:9], s[4:5], -1
+; SDAG-NEXT: s_xor_b64 s[10:11], s[4:5], -1
; SDAG-NEXT: v_or_b32_e32 v9, v0, v9
; SDAG-NEXT: v_sub_i32_e32 v0, vcc, 63, v8
; SDAG-NEXT: v_or_b32_e32 v10, v1, v10
@@ -105,14 +105,15 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_lshl_b64 v[8:9], v[16:17], v11
; SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc
; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v11
+; SDAG-NEXT: s_mov_b64 s[8:9], 0
; SDAG-NEXT: v_cndmask_b32_e64 v1, v1, v19, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v0, v0, v18, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v9, 0, v9, vcc
; SDAG-NEXT: v_mov_b32_e32 v10, 0
; SDAG-NEXT: v_mov_b32_e32 v11, 0
; SDAG-NEXT: v_cndmask_b32_e32 v8, 0, v8, vcc
-; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[10:11]
+; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
; SDAG-NEXT: s_cbranch_execz .LBB0_5
; SDAG-NEXT: ; %bb.2: ; %udiv-preheader4
; SDAG-NEXT: v_sub_i32_e32 v20, vcc, 64, v30
@@ -137,7 +138,6 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_addc_u32_e32 v37, vcc, -1, v3, vcc
; SDAG-NEXT: v_mov_b32_e32 v18, 0
; SDAG-NEXT: v_mov_b32_e32 v19, 0
-; SDAG-NEXT: s_mov_b64 s[4:5], 0
; SDAG-NEXT: v_mov_b32_e32 v22, 0
; SDAG-NEXT: v_mov_b32_e32 v23, 0
; SDAG-NEXT: v_mov_b32_e32 v11, 0
@@ -178,15 +178,15 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_and_b32_e32 v10, 1, v38
; SDAG-NEXT: v_or_b32_e32 v1, v19, v1
; SDAG-NEXT: v_or_b32_e32 v0, v18, v0
-; SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; SDAG-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SDAG-NEXT: v_mov_b32_e32 v23, v11
; SDAG-NEXT: v_mov_b32_e32 v22, v10
-; SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; SDAG-NEXT: s_andn2_b64 exec, exec, s[8:9]
; SDAG-NEXT: s_cbranch_execnz .LBB0_3
; SDAG-NEXT: ; %bb.4: ; %Flow13
-; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
-; SDAG-NEXT: .LBB0_5: ; %Flow14
; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
+; SDAG-NEXT: .LBB0_5: ; %Flow14
+; SDAG-NEXT: s_or_b64 exec, exec, s[10:11]
; SDAG-NEXT: v_lshl_b64 v[0:1], v[0:1], 1
; SDAG-NEXT: v_lshl_b64 v[2:3], v[8:9], 1
; SDAG-NEXT: v_lshrrev_b32_e32 v16, 31, v9
@@ -259,7 +259,7 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_mov_b32_e32 v23, v19
; SDAG-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[12:13]
-; SDAG-NEXT: s_movk_i32 s8, 0x7f
+; SDAG-NEXT: s_movk_i32 s10, 0x7f
; SDAG-NEXT: v_cndmask_b32_e32 v10, v11, v10, vcc
; SDAG-NEXT: v_and_b32_e32 v10, 1, v10
; SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 1, v10
@@ -280,7 +280,7 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_addc_u32_e32 v31, vcc, 0, v5, vcc
; SDAG-NEXT: v_addc_u32_e32 v32, vcc, 0, v12, vcc
; SDAG-NEXT: v_addc_u32_e64 v33, s[4:5], 0, v13, vcc
-; SDAG-NEXT: v_sub_i32_e32 v14, vcc, s8, v4
+; SDAG-NEXT: v_sub_i32_e32 v14, vcc, s10, v4
; SDAG-NEXT: v_sub_i32_e32 v5, vcc, 64, v14
; SDAG-NEXT: v_lshl_b64 v[10:11], v[6:7], v14
; SDAG-NEXT: v_lshr_b64 v[12:13], v[8:9], v5
@@ -288,20 +288,21 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_lshl_b64 v[4:5], v[8:9], v4
; SDAG-NEXT: v_or_b32_e32 v11, v11, v13
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
-; SDAG-NEXT: s_xor_b64 s[8:9], s[4:5], -1
+; SDAG-NEXT: s_xor_b64 s[10:11], s[4:5], -1
; SDAG-NEXT: v_or_b32_e32 v10, v10, v12
; SDAG-NEXT: v_cndmask_b32_e32 v5, v5, v11, vcc
; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
; SDAG-NEXT: v_cndmask_b32_e64 v11, v5, v7, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v10, v4, v10, vcc
; SDAG-NEXT: v_lshl_b64 v[4:5], v[8:9], v14
+; SDAG-NEXT: s_mov_b64 s[8:9], 0
; SDAG-NEXT: v_cndmask_b32_e64 v10, v10, v6, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
; SDAG-NEXT: v_mov_b32_e32 v12, 0
; SDAG-NEXT: v_mov_b32_e32 v13, 0
; SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
-; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[10:11]
+; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
; SDAG-NEXT: s_cbranch_execz .LBB0_11
; SDAG-NEXT: ; %bb.8: ; %udiv-preheader
; SDAG-NEXT: v_sub_i32_e32 v14, vcc, 64, v30
@@ -326,7 +327,6 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_addc_u32_e32 v37, vcc, -1, v3, vcc
; SDAG-NEXT: v_mov_b32_e32 v6, 0
; SDAG-NEXT: v_mov_b32_e32 v7, 0
-; SDAG-NEXT: s_mov_b64 s[10:11], 0
; SDAG-NEXT: v_mov_b32_e32 v14, 0
; SDAG-NEXT: v_mov_b32_e32 v15, 0
; SDAG-NEXT: v_mov_b32_e32 v13, 0
@@ -366,16 +366,16 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_subb_u32_e32 v16, vcc, v16, v39, vcc
; SDAG-NEXT: v_or_b32_e32 v11, v7, v11
; SDAG-NEXT: v_or_b32_e32 v10, v6, v10
-; SDAG-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11]
+; SDAG-NEXT: s_or_b64 s[8:9], s[4:5], s[8:9]
; SDAG-NEXT: v_mov_b32_e32 v15, v13
; SDAG-NEXT: v_mov_b32_e32 v14, v12
; SDAG-NEXT: v_subb_u32_e32 v17, vcc, v17, v38, vcc
-; SDAG-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; SDAG-NEXT: s_andn2_b64 exec, exec, s[8:9]
; SDAG-NEXT: s_cbranch_execnz .LBB0_9
; SDAG-NEXT: ; %bb.10: ; %Flow
-; SDAG-NEXT: s_or_b64 exec, exec, s[10:11]
-; SDAG-NEXT: .LBB0_11: ; %Flow11
; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
+; SDAG-NEXT: .LBB0_11: ; %Flow11
+; SDAG-NEXT: s_or_b64 exec, exec, s[10:11]
; SDAG-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
; SDAG-NEXT: v_lshl_b64 v[2:3], v[4:5], 1
; SDAG-NEXT: v_lshrrev_b32_e32 v6, 31, v5
@@ -474,31 +474,31 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[8:9]
; GISEL-NEXT: v_cmp_eq_u64_e64 s[8:9], 0, v[10:11]
-; GISEL-NEXT: v_mov_b32_e32 v26, v24
+; GISEL-NEXT: s_movk_i32 s10, 0x7f
; GISEL-NEXT: v_cndmask_b32_e64 v0, v1, v0, s[8:9]
; GISEL-NEXT: v_and_b32_e32 v0, 1, v0
; GISEL-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v0
; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[10:11], s[4:5], -1
+; GISEL-NEXT: s_xor_b64 s[8:9], s[4:5], -1
; GISEL-NEXT: v_cndmask_b32_e64 v1, v19, 0, s[4:5]
-; GISEL-NEXT: s_movk_i32 s8, 0x7f
; GISEL-NEXT: v_cndmask_b32_e64 v0, v18, 0, s[4:5]
-; GISEL-NEXT: s_and_b64 s[10:11], s[10:11], s[6:7]
+; GISEL-NEXT: s_and_b64 s[8:9], s[8:9], s[6:7]
+; GISEL-NEXT: v_mov_b32_e32 v26, v24
; GISEL-NEXT: v_mov_b32_e32 v27, v25
; GISEL-NEXT: v_cndmask_b32_e64 v20, v17, 0, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e64 v21, v16, 0, s[4:5]
-; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[10:11]
+; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[8:9]
; GISEL-NEXT: s_cbranch_execz .LBB0_6
; GISEL-NEXT: ; %bb.1: ; %udiv-bb15
; GISEL-NEXT: v_add_i32_e32 v30, vcc, 1, v8
; GISEL-NEXT: v_addc_u32_e32 v31, vcc, 0, v9, vcc
; GISEL-NEXT: v_addc_u32_e32 v32, vcc, 0, v10, vcc
; GISEL-NEXT: v_addc_u32_e64 v33, s[4:5], 0, v11, vcc
-; GISEL-NEXT: v_sub_i32_e32 v11, vcc, s8, v8
+; GISEL-NEXT: v_sub_i32_e32 v11, vcc, s10, v8
; GISEL-NEXT: v_sub_i32_e32 v9, vcc, 64, v11
; GISEL-NEXT: v_lshl_b64 v[0:1], v[18:19], v11
; GISEL-NEXT: v_lshr_b64 v[9:10], v[16:17], v9
-; GISEL-NEXT: s_xor_b64 s[8:9], s[4:5], -1
+; GISEL-NEXT: s_xor_b64 s[10:11], s[4:5], -1
; GISEL-NEXT: v_or_b32_e32 v9, v0, v9
; GISEL-NEXT: v_sub_i32_e32 v0, vcc, 63, v8
; GISEL-NEXT: v_or_b32_e32 v10, v1, v10
@@ -508,14 +508,15 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_lshl_b64 v[8:9], v[16:17], v11
; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc
; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v11
+; GISEL-NEXT: s_mov_b64 s[8:9], 0
; GISEL-NEXT: v_cndmask_b32_e64 v1, v1, v19, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e64 v0, v0, v18, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e32 v9, 0, v9, vcc
; GISEL-NEXT: v_mov_b32_e32 v10, 0
; GISEL-NEXT: v_mov_b32_e32 v11, 0
; GISEL-NEXT: v_cndmask_b32_e32 v8, 0, v8, vcc
-; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[10:11]
+; GISEL-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
; GISEL-NEXT: s_cbranch_execz .LBB0_5
; GISEL-NEXT: ; %bb.2: ; %udiv-preheader4
; GISEL-NEXT: v_sub_i32_e32 v20, vcc, 64, v30
@@ -540,7 +541,6 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_addc_u32_e32 v37, vcc, -1, v3, vcc
; GISEL-NEXT: v_mov_b32_e32 v18, 0
; GISEL-NEXT: v_mov_b32_e32 v19, 0
-; GISEL-NEXT: s_mov_b64 s[4:5], 0
; GISEL-NEXT: v_mov_b32_e32 v22, 0
; GISEL-NEXT: v_mov_b32_e32 v23, 0
; GISEL-NEXT: v_mov_b32_e32 v11, 0
@@ -581,15 +581,15 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_and_b32_e32 v10, 1, v38
; GISEL-NEXT: v_or_b32_e32 v1, v19, v1
; GISEL-NEXT: v_or_b32_e32 v0, v18, v0
-; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GISEL-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GISEL-NEXT: v_mov_b32_e32 v23, v11
; GISEL-NEXT: v_mov_b32_e32 v22, v10
-; GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GISEL-NEXT: s_cbranch_execnz .LBB0_3
; GISEL-NEXT: ; %bb.4: ; %Flow13
-; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
-; GISEL-NEXT: .LBB0_5: ; %Flow14
; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
+; GISEL-NEXT: .LBB0_5: ; %Flow14
+; GISEL-NEXT: s_or_b64 exec, exec, s[10:11]
; GISEL-NEXT: v_lshl_b64 v[0:1], v[0:1], 1
; GISEL-NEXT: v_lshl_b64 v[2:3], v[8:9], 1
; GISEL-NEXT: v_lshrrev_b32_e32 v16, 31, v9
@@ -662,7 +662,7 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_mov_b32_e32 v23, v19
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[12:13]
-; GISEL-NEXT: s_movk_i32 s8, 0x7f
+; GISEL-NEXT: s_movk_i32 s10, 0x7f
; GISEL-NEXT: v_cndmask_b32_e32 v10, v11, v10, vcc
; GISEL-NEXT: v_and_b32_e32 v10, 1, v10
; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 1, v10
@@ -683,7 +683,7 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_addc_u32_e32 v31, vcc, 0, v5, vcc
; GISEL-NEXT: v_addc_u32_e32 v32, vcc, 0, v12, vcc
; GISEL-NEXT: v_addc_u32_e64 v33, s[4:5], 0, v13, vcc
-; GISEL-NEXT: v_sub_i32_e32 v14, vcc, s8, v4
+; GISEL-NEXT: v_sub_i32_e32 v14, vcc, s10, v4
; GISEL-NEXT: v_sub_i32_e32 v5, vcc, 64, v14
; GISEL-NEXT: v_lshl_b64 v[10:11], v[6:7], v14
; GISEL-NEXT: v_lshr_b64 v[12:13], v[8:9], v5
@@ -691,20 +691,21 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_lshl_b64 v[4:5], v[8:9], v4
; GISEL-NEXT: v_or_b32_e32 v11, v11, v13
; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
-; GISEL-NEXT: s_xor_b64 s[8:9], s[4:5], -1
+; GISEL-NEXT: s_xor_b64 s[10:11], s[4:5], -1
; GISEL-NEXT: v_or_b32_e32 v10, v10, v12
; GISEL-NEXT: v_cndmask_b32_e32 v5, v5, v11, vcc
; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
; GISEL-NEXT: v_cndmask_b32_e64 v11, v5, v7, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e32 v10, v4, v10, vcc
; GISEL-NEXT: v_lshl_b64 v[4:5], v[8:9], v14
+; GISEL-NEXT: s_mov_b64 s[8:9], 0
; GISEL-NEXT: v_cndmask_b32_e64 v10, v10, v6, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
; GISEL-NEXT: v_mov_b32_e32 v12, 0
; GISEL-NEXT: v_mov_b32_e32 v13, 0
; GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[10:11]
+; GISEL-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
; GISEL-NEXT: s_cbranch_execz .LBB0_11
; GISEL-NEXT: ; %bb.8: ; %udiv-preheader
; GISEL-NEXT: v_sub_i32_e32 v14, vcc, 64, v30
@@ -729,7 +730,6 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_addc_u32_e32 v37, vcc, -1, v3, vcc
; GISEL-NEXT: v_mov_b32_e32 v6, 0
; GISEL-NEXT: v_mov_b32_e32 v7, 0
-; GISEL-NEXT: s_mov_b64 s[10:11], 0
; GISEL-NEXT: v_mov_b32_e32 v14, 0
; GISEL-NEXT: v_mov_b32_e32 v15, 0
; GISEL-NEXT: v_mov_b32_e32 v13, 0
@@ -769,16 +769,16 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_subb_u32_e32 v16, vcc, v16, v39, vcc
; GISEL-NEXT: v_or_b32_e32 v11, v7, v11
; GISEL-NEXT: v_or_b32_e32 v10, v6, v10
-; GISEL-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11]
+; GISEL-NEXT: s_or_b64 s[8:9], s[4:5], s[8:9]
; GISEL-NEXT: v_mov_b32_e32 v15, v13
; GISEL-NEXT: v_mov_b32_e32 v14, v12
; GISEL-NEXT: v_subb_u32_e32 v17, vcc, v17, v38, vcc
-; GISEL-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GISEL-NEXT: s_cbranch_execnz .LBB0_9
; GISEL-NEXT: ; %bb.10: ; %Flow
-; GISEL-NEXT: s_or_b64 exec, exec, s[10:11]
-; GISEL-NEXT: .LBB0_11: ; %Flow11
; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
+; GISEL-NEXT: .LBB0_11: ; %Flow11
+; GISEL-NEXT: s_or_b64 exec, exec, s[10:11]
; GISEL-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
; GISEL-NEXT: v_lshl_b64 v[2:3], v[4:5], 1
; GISEL-NEXT: v_lshrrev_b32_e32 v6, 31, v5
@@ -923,29 +923,29 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s[8:9]
; SDAG-NEXT: v_cmp_eq_u64_e64 s[8:9], 0, v[22:23]
+; SDAG-NEXT: s_movk_i32 s10, 0x7f
; SDAG-NEXT: v_cndmask_b32_e64 v2, v3, v2, s[8:9]
; SDAG-NEXT: v_and_b32_e32 v2, 1, v2
; SDAG-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v2
; SDAG-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
-; SDAG-NEXT: s_xor_b64 s[10:11], s[4:5], -1
+; SDAG-NEXT: s_xor_b64 s[8:9], s[4:5], -1
; SDAG-NEXT: v_cndmask_b32_e64 v3, v19, 0, s[4:5]
-; SDAG-NEXT: s_movk_i32 s8, 0x7f
; SDAG-NEXT: v_cndmask_b32_e64 v2, v18, 0, s[4:5]
-; SDAG-NEXT: s_and_b64 s[10:11], s[10:11], s[6:7]
+; SDAG-NEXT: s_and_b64 s[8:9], s[8:9], s[6:7]
; SDAG-NEXT: v_cndmask_b32_e64 v16, v1, 0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v17, v0, 0, s[4:5]
-; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[10:11]
+; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[8:9]
; SDAG-NEXT: s_cbranch_execz .LBB2_6
; SDAG-NEXT: ; %bb.1: ; %udiv-bb15
; SDAG-NEXT: v_add_i32_e32 v26, vcc, 1, v20
; SDAG-NEXT: v_addc_u32_e32 v27, vcc, 0, v21, vcc
; SDAG-NEXT: v_addc_u32_e32 v28, vcc, 0, v22, vcc
; SDAG-NEXT: v_addc_u32_e64 v29, s[4:5], 0, v23, vcc
-; SDAG-NEXT: v_sub_i32_e32 v21, vcc, s8, v20
+; SDAG-NEXT: v_sub_i32_e32 v21, vcc, s10, v20
; SDAG-NEXT: v_sub_i32_e32 v16, vcc, 64, v21
; SDAG-NEXT: v_lshl_b64 v[2:3], v[18:19], v21
; SDAG-NEXT: v_lshr_b64 v[16:17], v[0:1], v16
-; SDAG-NEXT: s_xor_b64 s[8:9], s[4:5], -1
+; SDAG-NEXT: s_xor_b64 s[10:11], s[4:5], -1
; SDAG-NEXT: v_or_b32_e32 v16, v2, v16
; SDAG-NEXT: v_sub_i32_e32 v2, vcc, 63, v20
; SDAG-NEXT: v_or_b32_e32 v17, v3, v17
@@ -955,14 +955,15 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v16, vcc
; SDAG-NEXT: v_lshl_b64 v[16:17], v[0:1], v21
; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v21
+; SDAG-NEXT: s_mov_b64 s[8:9], 0
; SDAG-NEXT: v_cndmask_b32_e64 v3, v3, v19, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v2, v2, v18, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v17, 0, v17, vcc
; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: v_mov_b32_e32 v21, 0
; SDAG-NEXT: v_cndmask_b32_e32 v16, 0, v16, vcc
-; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[10:11]
+; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
; SDAG-NEXT: s_cbranch_execz .LBB2_5
; SDAG-NEXT: ; %bb.2: ; %udiv-preheader4
; SDAG-NEXT: v_sub_i32_e32 v22, vcc, 64, v26
@@ -987,7 +988,6 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_addc_u32_e32 v33, vcc, -1, v11, vcc
; SDAG-NEXT: v_mov_b32_e32 v0, 0
; SDAG-NEXT: v_mov_b32_e32 v1, 0
-; SDAG-NEXT: s_mov_b64 s[4:5], 0
; SDAG-NEXT: v_mov_b32_e32 v24, 0
; SDAG-NEXT: v_mov_b32_e32 v25, 0
; SDAG-NEXT: v_mov_b32_e32 v21, 0
@@ -1028,15 +1028,15 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_and_b32_e32 v20, 1, v34
; SDAG-NEXT: v_or_b32_e32 v3, v1, v3
; SDAG-NEXT: v_or_b32_e32 v2, v0, v2
-; SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; SDAG-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SDAG-NEXT: v_mov_b32_e32 v25, v21
; SDAG-NEXT: v_mov_b32_e32 v24, v20
-; SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; SDAG-NEXT: s_andn2_b64 exec, exec, s[8:9]
; SDAG-NEXT: s_cbranch_execnz .LBB2_3
; SDAG-NEXT: ; %bb.4: ; %Flow13
-; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
-; SDAG-NEXT: .LBB2_5: ; %Flow14
; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
+; SDAG-NEXT: .LBB2_5: ; %Flow14
+; SDAG-NEXT: s_or_b64 exec, exec, s[10:11]
; SDAG-NEXT: v_lshl_b64 v[2:3], v[2:3], 1
; SDAG-NEXT: v_lshl_b64 v[0:1], v[16:17], 1
; SDAG-NEXT: v_lshrrev_b32_e32 v8, 31, v17
@@ -1076,7 +1076,7 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_add_i32_e32 v9, vcc, 64, v9
; SDAG-NEXT: v_addc_u32_e64 v10, s[6:7], 0, 0, vcc
; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; SDAG-NEXT: s_movk_i32 s8, 0x7f
+; SDAG-NEXT: s_movk_i32 s10, 0x7f
; SDAG-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v10, v10, 0, vcc
; SDAG-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
@@ -1110,7 +1110,7 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_addc_u32_e32 v24, vcc, 0, v1, vcc
; SDAG-NEXT: v_addc_u32_e32 v25, vcc, 0, v18, vcc
; SDAG-NEXT: v_addc_u32_e64 v26, s[4:5], 0, v19, vcc
-; SDAG-NEXT: v_sub_i32_e32 v8, vcc, s8, v0
+; SDAG-NEXT: v_sub_i32_e32 v8, vcc, s10, v0
; SDAG-NEXT: v_sub_i32_e32 v1, vcc, 64, v8
; SDAG-NEXT: v_lshl_b64 v[10:11], v[6:7], v8
; SDAG-NEXT: v_lshr_b64 v[18:19], v[4:5], v1
@@ -1122,16 +1122,17 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc
; SDAG-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc
; SDAG-NEXT: v_lshl_b64 v[10:11], v[4:5], v8
-; SDAG-NEXT: s_xor_b64 s[8:9], s[4:5], -1
+; SDAG-NEXT: s_xor_b64 s[10:11], s[4:5], -1
; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v8
+; SDAG-NEXT: s_mov_b64 s[8:9], 0
; SDAG-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v11, 0, v11, vcc
; SDAG-NEXT: v_mov_b32_e32 v18, 0
; SDAG-NEXT: v_mov_b32_e32 v19, 0
; SDAG-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc
-; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[10:11]
+; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
; SDAG-NEXT: s_cbranch_execz .LBB2_11
; SDAG-NEXT: ; %bb.8: ; %udiv-preheader
; SDAG-NEXT: v_sub_i32_e32 v8, vcc, 64, v9
@@ -1156,7 +1157,6 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_addc_u32_e32 v29, vcc, -1, v15, vcc
; SDAG-NEXT: v_mov_b32_e32 v4, 0
; SDAG-NEXT: v_mov_b32_e32 v5, 0
-; SDAG-NEXT: s_mov_b64 s[10:11], 0
; SDAG-NEXT: v_mov_b32_e32 v22, 0
; SDAG-NEXT: v_mov_b32_e32 v23, 0
; SDAG-NEXT: v_mov_b32_e32 v19, 0
@@ -1197,15 +1197,15 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_or_b32_e32 v1, v5, v1
; SDAG-NEXT: v_or_b32_e32 v0, v4, v0
; SDAG-NEXT: v_subb_u32_e32 v21, vcc, v21, v30, vcc
-; SDAG-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11]
+; SDAG-NEXT: s_or_b64 s[8:9], s[4:5], s[8:9]
; SDAG-NEXT: v_mov_b32_e32 v23, v19
; SDAG-NEXT: v_mov_b32_e32 v22, v18
-; SDAG-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; SDAG-NEXT: s_andn2_b64 exec, exec, s[8:9]
; SDAG-NEXT: s_cbranch_execnz .LBB2_9
; SDAG-NEXT: ; %bb.10: ; %Flow
-; SDAG-NEXT: s_or_b64 exec, exec, s[10:11]
-; SDAG-NEXT: .LBB2_11: ; %Flow11
; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
+; SDAG-NEXT: .LBB2_11: ; %Flow11
+; SDAG-NEXT: s_or_b64 exec, exec, s[10:11]
; SDAG-NEXT: v_lshl_b64 v[8:9], v[0:1], 1
; SDAG-NEXT: v_lshl_b64 v[0:1], v[10:11], 1
; SDAG-NEXT: v_lshrrev_b32_e32 v4, 31, v11
@@ -1274,29 +1274,29 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, s[8:9]
; GISEL-NEXT: v_cmp_eq_u64_e64 s[8:9], 0, v[22:23]
+; GISEL-NEXT: s_movk_i32 s10, 0x7f
; GISEL-NEXT: v_cndmask_b32_e64 v2, v3, v2, s[8:9]
; GISEL-NEXT: v_and_b32_e32 v2, 1, v2
; GISEL-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v2
; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[10:11], s[4:5], -1
+; GISEL-NEXT: s_xor_b64 s[8:9], s[4:5], -1
; GISEL-NEXT: v_cndmask_b32_e64 v3, v19, 0, s[4:5]
-; GISEL-NEXT: s_movk_i32 s8, 0x7f
; GISEL-NEXT: v_cndmask_b32_e64 v2, v18, 0, s[4:5]
-; GISEL-NEXT: s_and_b64 s[10:11], s[10:11], s[6:7]
+; GISEL-NEXT: s_and_b64 s[8:9], s[8:9], s[6:7]
; GISEL-NEXT: v_cndmask_b32_e64 v16, v1, 0, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e64 v17, v0, 0, s[4:5]
-; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[10:11]
+; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[8:9]
; GISEL-NEXT: s_cbranch_execz .LBB2_6
; GISEL-NEXT: ; %bb.1: ; %udiv-bb15
; GISEL-NEXT: v_add_i32_e32 v26, vcc, 1, v20
; GISEL-NEXT: v_addc_u32_e32 v27, vcc, 0, v21, vcc
; GISEL-NEXT: v_addc_u32_e32 v28, vcc, 0, v22, vcc
; GISEL-NEXT: v_addc_u32_e64 v29, s[4:5], 0, v23, vcc
-; GISEL-NEXT: v_sub_i32_e32 v21, vcc, s8, v20
+; GISEL-NEXT: v_sub_i32_e32 v21, vcc, s10, v20
; GISEL-NEXT: v_sub_i32_e32 v16, vcc, 64, v21
; GISEL-NEXT: v_lshl_b64 v[2:3], v[18:19], v21
; GISEL-NEXT: v_lshr_b64 v[16:17], v[0:1], v16
-; GISEL-NEXT: s_xor_b64 s[8:9], s[4:5], -1
+; GISEL-NEXT: s_xor_b64 s[10:11], s[4:5], -1
; GISEL-NEXT: v_or_b32_e32 v16, v2, v16
; GISEL-NEXT: v_sub_i32_e32 v2, vcc, 63, v20
; GISEL-NEXT: v_or_b32_e32 v17, v3, v17
@@ -1306,14 +1306,15 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v16, vcc
; GISEL-NEXT: v_lshl_b64 v[16:17], v[0:1], v21
; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v21
+; GISEL-NEXT: s_mov_b64 s[8:9], 0
; GISEL-NEXT: v_cndmask_b32_e64 v3, v3, v19, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e64 v2, v2, v18, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e32 v17, 0, v17, vcc
; GISEL-NEXT: v_mov_b32_e32 v20, 0
; GISEL-NEXT: v_mov_b32_e32 v21, 0
; GISEL-NEXT: v_cndmask_b32_e32 v16, 0, v16, vcc
-; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[10:11]
+; GISEL-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
; GISEL-NEXT: s_cbranch_execz .LBB2_5
; GISEL-NEXT: ; %bb.2: ; %udiv-preheader4
; GISEL-NEXT: v_sub_i32_e32 v22, vcc, 64, v26
@@ -1338,7 +1339,6 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_addc_u32_e32 v33, vcc, -1, v11, vcc
; GISEL-NEXT: v_mov_b32_e32 v0, 0
; GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GISEL-NEXT: s_mov_b64 s[4:5], 0
; GISEL-NEXT: v_mov_b32_e32 v24, 0
; GISEL-NEXT: v_mov_b32_e32 v25, 0
; GISEL-NEXT: v_mov_b32_e32 v21, 0
@@ -1379,15 +1379,15 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_and_b32_e32 v20, 1, v34
; GISEL-NEXT: v_or_b32_e32 v3, v1, v3
; GISEL-NEXT: v_or_b32_e32 v2, v0, v2
-; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GISEL-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GISEL-NEXT: v_mov_b32_e32 v25, v21
; GISEL-NEXT: v_mov_b32_e32 v24, v20
-; GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GISEL-NEXT: s_cbranch_execnz .LBB2_3
; GISEL-NEXT: ; %bb.4: ; %Flow13
-; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
-; GISEL-NEXT: .LBB2_5: ; %Flow14
; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
+; GISEL-NEXT: .LBB2_5: ; %Flow14
+; GISEL-NEXT: s_or_b64 exec, exec, s[10:11]
; GISEL-NEXT: v_lshl_b64 v[2:3], v[2:3], 1
; GISEL-NEXT: v_lshl_b64 v[0:1], v[16:17], 1
; GISEL-NEXT: v_lshrrev_b32_e32 v8, 31, v17
@@ -1427,7 +1427,7 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_add_i32_e32 v9, vcc, 64, v9
; GISEL-NEXT: v_addc_u32_e64 v10, s[6:7], 0, 0, vcc
; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GISEL-NEXT: s_movk_i32 s8, 0x7f
+; GISEL-NEXT: s_movk_i32 s10, 0x7f
; GISEL-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc
; GISEL-NEXT: v_cndmask_b32_e64 v10, v10, 0, vcc
; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
@@ -1461,7 +1461,7 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_addc_u32_e32 v24, vcc, 0, v1, vcc
; GISEL-NEXT: v_addc_u32_e32 v25, vcc, 0, v18, vcc
; GISEL-NEXT: v_addc_u32_e64 v26, s[4:5], 0, v19, vcc
-; GISEL-NEXT: v_sub_i32_e32 v8, vcc, s8, v0
+; GISEL-NEXT: v_sub_i32_e32 v8, vcc, s10, v0
; GISEL-NEXT: v_sub_i32_e32 v1, vcc, 64, v8
; GISEL-NEXT: v_lshl_b64 v[10:11], v[6:7], v8
; GISEL-NEXT: v_lshr_b64 v[18:19], v[4:5], v1
@@ -1473,16 +1473,17 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc
; GISEL-NEXT: v_lshl_b64 v[10:11], v[4:5], v8
-; GISEL-NEXT: s_xor_b64 s[8:9], s[4:5], -1
+; GISEL-NEXT: s_xor_b64 s[10:11], s[4:5], -1
; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v8
+; GISEL-NEXT: s_mov_b64 s[8:9], 0
; GISEL-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e32 v11, 0, v11, vcc
; GISEL-NEXT: v_mov_b32_e32 v18, 0
; GISEL-NEXT: v_mov_b32_e32 v19, 0
; GISEL-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc
-; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[10:11]
+; GISEL-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
; GISEL-NEXT: s_cbranch_execz .LBB2_11
; GISEL-NEXT: ; %bb.8: ; %udiv-preheader
; GISEL-NEXT: v_sub_i32_e32 v8, vcc, 64, v9
@@ -1507,7 +1508,6 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_addc_u32_e32 v29, vcc, -1, v15, vcc
; GISEL-NEXT: v_mov_b32_e32 v4, 0
; GISEL-NEXT: v_mov_b32_e32 v5, 0
-; GISEL-NEXT: s_mov_b64 s[10:11], 0
; GISEL-NEXT: v_mov_b32_e32 v22, 0
; GISEL-NEXT: v_mov_b32_e32 v23, 0
; GISEL-NEXT: v_mov_b32_e32 v19, 0
@@ -1548,15 +1548,15 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_or_b32_e32 v1, v5, v1
; GISEL-NEXT: v_or_b32_e32 v0, v4, v0
; GISEL-NEXT: v_subb_u32_e32 v21, vcc, v21, v30, vcc
-; GISEL-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11]
+; GISEL-NEXT: s_or_b64 s[8:9], s[4:5], s[8:9]
; GISEL-NEXT: v_mov_b32_e32 v23, v19
; GISEL-NEXT: v_mov_b32_e32 v22, v18
-; GISEL-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GISEL-NEXT: s_cbranch_execnz .LBB2_9
; GISEL-NEXT: ; %bb.10: ; %Flow
-; GISEL-NEXT: s_or_b64 exec, exec, s[10:11]
-; GISEL-NEXT: .LBB2_11: ; %Flow11
; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
+; GISEL-NEXT: .LBB2_11: ; %Flow11
+; GISEL-NEXT: s_or_b64 exec, exec, s[10:11]
; GISEL-NEXT: v_lshl_b64 v[8:9], v[0:1], 1
; GISEL-NEXT: v_lshl_b64 v[0:1], v[10:11], 1
; GISEL-NEXT: v_lshrrev_b32_e32 v4, 31, v11
@@ -1680,30 +1680,30 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v11, 0, 1, s[8:9]
; SDAG-NEXT: v_cmp_eq_u64_e64 s[8:9], 0, v[18:19]
-; SDAG-NEXT: v_mov_b32_e32 v31, v28
+; SDAG-NEXT: s_movk_i32 s10, 0x7f
; SDAG-NEXT: v_cndmask_b32_e64 v10, v11, v10, s[8:9]
; SDAG-NEXT: v_and_b32_e32 v10, 1, v10
; SDAG-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v10
; SDAG-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
-; SDAG-NEXT: s_xor_b64 s[10:11], s[4:5], -1
+; SDAG-NEXT: s_xor_b64 s[8:9], s[4:5], -1
; SDAG-NEXT: v_cndmask_b32_e64 v11, v3, 0, s[4:5]
-; SDAG-NEXT: s_movk_i32 s8, 0x7f
; SDAG-NEXT: v_cndmask_b32_e64 v10, v2, 0, s[4:5]
-; SDAG-NEXT: s_and_b64 s[10:11], s[10:11], s[6:7]
+; SDAG-NEXT: s_and_b64 s[8:9], s[8:9], s[6:7]
+; SDAG-NEXT: v_mov_b32_e32 v31, v28
; SDAG-NEXT: v_cndmask_b32_e64 v35, v1, 0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v37, v0, 0, s[4:5]
-; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[10:11]
+; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[8:9]
; SDAG-NEXT: s_cbranch_execz .LBB4_6
; SDAG-NEXT: ; %bb.1: ; %udiv-bb15
; SDAG-NEXT: v_add_i32_e32 v32, vcc, 1, v16
; SDAG-NEXT: v_addc_u32_e32 v33, vcc, 0, v17, vcc
; SDAG-NEXT: v_addc_u32_e32 v34, vcc, 0, v18, vcc
; SDAG-NEXT: v_addc_u32_e64 v35, s[4:5], 0, v19, vcc
-; SDAG-NEXT: v_sub_i32_e32 v19, vcc, s8, v16
+; SDAG-NEXT: v_sub_i32_e32 v19, vcc, s10, v16
; SDAG-NEXT: v_sub_i32_e32 v17, vcc, 64, v19
; SDAG-NEXT: v_lshl_b64 v[10:11], v[2:3], v19
; SDAG-NEXT: v_lshr_b64 v[17:18], v[0:1], v17
-; SDAG-NEXT: s_xor_b64 s[8:9], s[4:5], -1
+; SDAG-NEXT: s_xor_b64 s[10:11], s[4:5], -1
; SDAG-NEXT: v_or_b32_e32 v17, v10, v17
; SDAG-NEXT: v_sub_i32_e32 v10, vcc, 63, v16
; SDAG-NEXT: v_or_b32_e32 v18, v11, v18
@@ -1713,14 +1713,15 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_lshl_b64 v[16:17], v[0:1], v19
; SDAG-NEXT: v_cndmask_b32_e32 v11, v11, v18, vcc
; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v19
+; SDAG-NEXT: s_mov_b64 s[8:9], 0
; SDAG-NEXT: v_cndmask_b32_e64 v11, v11, v3, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v10, v10, v2, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v17, 0, v17, vcc
; SDAG-NEXT: v_mov_b32_e32 v18, 0
; SDAG-NEXT: v_mov_b32_e32 v19, 0
; SDAG-NEXT: v_cndmask_b32_e32 v16, 0, v16, vcc
-; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[10:11]
+; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
; SDAG-NEXT: s_cbranch_execz .LBB4_5
; SDAG-NEXT: ; %bb.2: ; %udiv-preheader4
; SDAG-NEXT: v_sub_i32_e32 v20, vcc, 64, v32
@@ -1745,7 +1746,6 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_addc_u32_e32 v39, vcc, -1, v9, vcc
; SDAG-NEXT: v_mov_b32_e32 v22, 0
; SDAG-NEXT: v_mov_b32_e32 v23, 0
-; SDAG-NEXT: s_mov_b64 s[4:5], 0
; SDAG-NEXT: v_mov_b32_e32 v26, 0
; SDAG-NEXT: v_mov_b32_e32 v27, 0
; SDAG-NEXT: v_mov_b32_e32 v19, 0
@@ -1786,15 +1786,15 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_and_b32_e32 v18, 1, v48
; SDAG-NEXT: v_or_b32_e32 v11, v23, v11
; SDAG-NEXT: v_or_b32_e32 v10, v22, v10
-; SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; SDAG-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SDAG-NEXT: v_mov_b32_e32 v27, v19
; SDAG-NEXT: v_mov_b32_e32 v26, v18
-; SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; SDAG-NEXT: s_andn2_b64 exec, exec, s[8:9]
; SDAG-NEXT: s_cbranch_execnz .LBB4_3
; SDAG-NEXT: ; %bb.4: ; %Flow13
-; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
-; SDAG-NEXT: .LBB4_5: ; %Flow14
; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
+; SDAG-NEXT: .LBB4_5: ; %Flow14
+; SDAG-NEXT: s_or_b64 exec, exec, s[10:11]
; SDAG-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
; SDAG-NEXT: v_lshrrev_b32_e32 v20, 31, v17
; SDAG-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
@@ -1864,7 +1864,7 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_or_b32_e32 v21, v15, v19
; SDAG-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[18:19]
-; SDAG-NEXT: s_movk_i32 s8, 0x7f
+; SDAG-NEXT: s_movk_i32 s10, 0x7f
; SDAG-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc
; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[18:19]
; SDAG-NEXT: v_cndmask_b32_e32 v16, v17, v16, vcc
@@ -1887,7 +1887,7 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_addc_u32_e32 v39, vcc, 0, v15, vcc
; SDAG-NEXT: v_addc_u32_e32 v48, vcc, 0, v18, vcc
; SDAG-NEXT: v_addc_u32_e64 v49, s[4:5], 0, v19, vcc
-; SDAG-NEXT: v_sub_i32_e32 v19, vcc, s8, v14
+; SDAG-NEXT: v_sub_i32_e32 v19, vcc, s10, v14
; SDAG-NEXT: v_sub_i32_e32 v17, vcc, 64, v19
; SDAG-NEXT: v_lshl_b64 v[15:16], v[6:7], v19
; SDAG-NEXT: v_lshr_b64 v[17:18], v[4:5], v17
@@ -1896,19 +1896,20 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_or_b32_e32 v18, v15, v17
; SDAG-NEXT: v_lshl_b64 v[14:15], v[4:5], v14
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
-; SDAG-NEXT: s_xor_b64 s[8:9], s[4:5], -1
+; SDAG-NEXT: s_xor_b64 s[10:11], s[4:5], -1
; SDAG-NEXT: v_cndmask_b32_e32 v15, v15, v16, vcc
; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v19
; SDAG-NEXT: v_cndmask_b32_e64 v17, v15, v7, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v16, v14, v18, vcc
; SDAG-NEXT: v_lshl_b64 v[14:15], v[4:5], v19
+; SDAG-NEXT: s_mov_b64 s[8:9], 0
; SDAG-NEXT: v_cndmask_b32_e64 v16, v16, v6, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v15, 0, v15, vcc
; SDAG-NEXT: v_mov_b32_e32 v18, 0
; SDAG-NEXT: v_mov_b32_e32 v19, 0
; SDAG-NEXT: v_cndmask_b32_e32 v14, 0, v14, vcc
-; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[10:11]
+; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
; SDAG-NEXT: s_cbranch_execz .LBB4_11
; SDAG-NEXT: ; %bb.8: ; %udiv-preheader
; SDAG-NEXT: v_sub_i32_e32 v20, vcc, 64, v38
@@ -1933,7 +1934,6 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_addc_u32_e32 v53, vcc, -1, v13, vcc
; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: v_mov_b32_e32 v21, 0
-; SDAG-NEXT: s_mov_b64 s[10:11], 0
; SDAG-NEXT: v_mov_b32_e32 v22, 0
; SDAG-NEXT: v_mov_b32_e32 v23, 0
; SDAG-NEXT: v_mov_b32_e32 v19, 0
@@ -1973,16 +1973,16 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_subb_u32_e32 v26, vcc, v26, v55, vcc
; SDAG-NEXT: v_or_b32_e32 v17, v21, v17
; SDAG-NEXT: v_or_b32_e32 v16, v20, v16
-; SDAG-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11]
+; SDAG-NEXT: s_or_b64 s[8:9], s[4:5], s[8:9]
; SDAG-NEXT: v_mov_b32_e32 v23, v19
; SDAG-NEXT: v_mov_b32_e32 v22, v18
; SDAG-NEXT: v_subb_u32_e32 v27, vcc, v27, v54, vcc
-; SDAG-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; SDAG-NEXT: s_andn2_b64 exec, exec, s[8:9]
; SDAG-NEXT: s_cbranch_execnz .LBB4_9
; SDAG-NEXT: ; %bb.10: ; %Flow
-; SDAG-NEXT: s_or_b64 exec, exec, s[10:11]
-; SDAG-NEXT: .LBB4_11: ; %Flow11
; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
+; SDAG-NEXT: .LBB4_11: ; %Flow11
+; SDAG-NEXT: s_or_b64 exec, exec, s[10:11]
; SDAG-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
; SDAG-NEXT: v_lshrrev_b32_e32 v20, 31, v15
; SDAG-NEXT: v_lshl_b64 v[14:15], v[14:15], 1
@@ -2127,30 +2127,30 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, s[8:9]
; GISEL-NEXT: v_cmp_eq_u64_e64 s[8:9], 0, v[18:19]
-; GISEL-NEXT: v_mov_b32_e32 v31, v28
+; GISEL-NEXT: s_movk_i32 s10, 0x7f
; GISEL-NEXT: v_cndmask_b32_e64 v10, v11, v10, s[8:9]
; GISEL-NEXT: v_and_b32_e32 v10, 1, v10
; GISEL-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v10
; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[10:11], s[4:5], -1
+; GISEL-NEXT: s_xor_b64 s[8:9], s[4:5], -1
; GISEL-NEXT: v_cndmask_b32_e64 v11, v3, 0, s[4:5]
-; GISEL-NEXT: s_movk_i32 s8, 0x7f
; GISEL-NEXT: v_cndmask_b32_e64 v10, v2, 0, s[4:5]
-; GISEL-NEXT: s_and_b64 s[10:11], s[10:11], s[6:7]
+; GISEL-NEXT: s_and_b64 s[8:9], s[8:9], s[6:7]
+; GISEL-NEXT: v_mov_b32_e32 v31, v28
; GISEL-NEXT: v_cndmask_b32_e64 v35, v1, 0, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e64 v37, v0, 0, s[4:5]
-; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[10:11]
+; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[8:9]
; GISEL-NEXT: s_cbranch_execz .LBB4_6
; GISEL-NEXT: ; %bb.1: ; %udiv-bb15
; GISEL-NEXT: v_add_i32_e32 v32, vcc, 1, v16
; GISEL-NEXT: v_addc_u32_e32 v33, vcc, 0, v17, vcc
; GISEL-NEXT: v_addc_u32_e32 v34, vcc, 0, v18, vcc
; GISEL-NEXT: v_addc_u32_e64 v35, s[4:5], 0, v19, vcc
-; GISEL-NEXT: v_sub_i32_e32 v19, vcc, s8, v16
+; GISEL-NEXT: v_sub_i32_e32 v19, vcc, s10, v16
; GISEL-NEXT: v_sub_i32_e32 v17, vcc, 64, v19
; GISEL-NEXT: v_lshl_b64 v[10:11], v[2:3], v19
; GISEL-NEXT: v_lshr_b64 v[17:18], v[0:1], v17
-; GISEL-NEXT: s_xor_b64 s[8:9], s[4:5], -1
+; GISEL-NEXT: s_xor_b64 s[10:11], s[4:5], -1
; GISEL-NEXT: v_or_b32_e32 v17, v10, v17
; GISEL-NEXT: v_sub_i32_e32 v10, vcc, 63, v16
; GISEL-NEXT: v_or_b32_e32 v18, v11, v18
@@ -2160,14 +2160,15 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_lshl_b64 v[16:17], v[0:1], v19
; GISEL-NEXT: v_cndmask_b32_e32 v11, v11, v18, vcc
; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v19
+; GISEL-NEXT: s_mov_b64 s[8:9], 0
; GISEL-NEXT: v_cndmask_b32_e64 v11, v11, v3, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e64 v10, v10, v2, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e32 v17, 0, v17, vcc
; GISEL-NEXT: v_mov_b32_e32 v18, 0
; GISEL-NEXT: v_mov_b32_e32 v19, 0
; GISEL-NEXT: v_cndmask_b32_e32 v16, 0, v16, vcc
-; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[10:11]
+; GISEL-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
; GISEL-NEXT: s_cbranch_execz .LBB4_5
; GISEL-NEXT: ; %bb.2: ; %udiv-preheader4
; GISEL-NEXT: v_sub_i32_e32 v20, vcc, 64, v32
@@ -2192,7 +2193,6 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_addc_u32_e32 v39, vcc, -1, v9, vcc
; GISEL-NEXT: v_mov_b32_e32 v22, 0
; GISEL-NEXT: v_mov_b32_e32 v23, 0
-; GISEL-NEXT: s_mov_b64 s[4:5], 0
; GISEL-NEXT: v_mov_b32_e32 v26, 0
; GISEL-NEXT: v_mov_b32_e32 v27, 0
; GISEL-NEXT: v_mov_b32_e32 v19, 0
@@ -2233,15 +2233,15 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_and_b32_e32 v18, 1, v48
; GISEL-NEXT: v_or_b32_e32 v11, v23, v11
; GISEL-NEXT: v_or_b32_e32 v10, v22, v10
-; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GISEL-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GISEL-NEXT: v_mov_b32_e32 v27, v19
; GISEL-NEXT: v_mov_b32_e32 v26, v18
-; GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GISEL-NEXT: s_cbranch_execnz .LBB4_3
; GISEL-NEXT: ; %bb.4: ; %Flow13
-; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
-; GISEL-NEXT: .LBB4_5: ; %Flow14
; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
+; GISEL-NEXT: .LBB4_5: ; %Flow14
+; GISEL-NEXT: s_or_b64 exec, exec, s[10:11]
; GISEL-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
; GISEL-NEXT: v_lshrrev_b32_e32 v20, 31, v17
; GISEL-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
@@ -2311,7 +2311,7 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_or_b32_e32 v21, v15, v19
; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[18:19]
-; GISEL-NEXT: s_movk_i32 s8, 0x7f
+; GISEL-NEXT: s_movk_i32 s10, 0x7f
; GISEL-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc
; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[18:19]
; GISEL-NEXT: v_cndmask_b32_e32 v16, v17, v16, vcc
@@ -2334,7 +2334,7 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_addc_u32_e32 v39, vcc, 0, v15, vcc
; GISEL-NEXT: v_addc_u32_e32 v48, vcc, 0, v18, vcc
; GISEL-NEXT: v_addc_u32_e64 v49, s[4:5], 0, v19, vcc
-; GISEL-NEXT: v_sub_i32_e32 v19, vcc, s8, v14
+; GISEL-NEXT: v_sub_i32_e32 v19, vcc, s10, v14
; GISEL-NEXT: v_sub_i32_e32 v17, vcc, 64, v19
; GISEL-NEXT: v_lshl_b64 v[15:16], v[6:7], v19
; GISEL-NEXT: v_lshr_b64 v[17:18], v[4:5], v17
@@ -2343,19 +2343,20 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_or_b32_e32 v18, v15, v17
; GISEL-NEXT: v_lshl_b64 v[14:15], v[4:5], v14
; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
-; GISEL-NEXT: s_xor_b64 s[8:9], s[4:5], -1
+; GISEL-NEXT: s_xor_b64 s[10:11], s[4:5], -1
; GISEL-NEXT: v_cndmask_b32_e32 v15, v15, v16, vcc
; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v19
; GISEL-NEXT: v_cndmask_b32_e64 v17, v15, v7, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e32 v16, v14, v18, vcc
; GISEL-NEXT: v_lshl_b64 v[14:15], v[4:5], v19
+; GISEL-NEXT: s_mov_b64 s[8:9], 0
; GISEL-NEXT: v_cndmask_b32_e64 v16, v16, v6, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e32 v15, 0, v15, vcc
; GISEL-NEXT: v_mov_b32_e32 v18, 0
; GISEL-NEXT: v_mov_b32_e32 v19, 0
; GISEL-NEXT: v_cndmask_b32_e32 v14, 0, v14, vcc
-; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[10:11]
+; GISEL-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
; GISEL-NEXT: s_cbranch_execz .LBB4_11
; GISEL-NEXT: ; %bb.8: ; %udiv-preheader
; GISEL-NEXT: v_sub_i32_e32 v20, vcc, 64, v38
@@ -2380,7 +2381,6 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_addc_u32_e32 v53, vcc, -1, v13, vcc
; GISEL-NEXT: v_mov_b32_e32 v20, 0
; GISEL-NEXT: v_mov_b32_e32 v21, 0
-; GISEL-NEXT: s_mov_b64 s[10:11], 0
; GISEL-NEXT: v_mov_b32_e32 v22, 0
; GISEL-NEXT: v_mov_b32_e32 v23, 0
; GISEL-NEXT: v_mov_b32_e32 v19, 0
@@ -2420,16 +2420,16 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_subb_u32_e32 v26, vcc, v26, v55, vcc
; GISEL-NEXT: v_or_b32_e32 v17, v21, v17
; GISEL-NEXT: v_or_b32_e32 v16, v20, v16
-; GISEL-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11]
+; GISEL-NEXT: s_or_b64 s[8:9], s[4:5], s[8:9]
; GISEL-NEXT: v_mov_b32_e32 v23, v19
; GISEL-NEXT: v_mov_b32_e32 v22, v18
; GISEL-NEXT: v_subb_u32_e32 v27, vcc, v27, v54, vcc
-; GISEL-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GISEL-NEXT: s_cbranch_execnz .LBB4_9
; GISEL-NEXT: ; %bb.10: ; %Flow
-; GISEL-NEXT: s_or_b64 exec, exec, s[10:11]
-; GISEL-NEXT: .LBB4_11: ; %Flow11
; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
+; GISEL-NEXT: .LBB4_11: ; %Flow11
+; GISEL-NEXT: s_or_b64 exec, exec, s[10:11]
; GISEL-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
; GISEL-NEXT: v_lshrrev_b32_e32 v20, 31, v15
; GISEL-NEXT: v_lshl_b64 v[14:15], v[14:15], 1
@@ -2559,29 +2559,29 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v17, 0, 1, s[8:9]
; SDAG-NEXT: v_cmp_eq_u64_e64 s[8:9], 0, v[20:21]
+; SDAG-NEXT: s_movk_i32 s10, 0x7f
; SDAG-NEXT: v_cndmask_b32_e64 v16, v17, v16, s[8:9]
; SDAG-NEXT: v_and_b32_e32 v16, 1, v16
; SDAG-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v16
; SDAG-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
-; SDAG-NEXT: s_xor_b64 s[10:11], s[4:5], -1
+; SDAG-NEXT: s_xor_b64 s[8:9], s[4:5], -1
; SDAG-NEXT: v_cndmask_b32_e64 v17, v3, 0, s[4:5]
-; SDAG-NEXT: s_movk_i32 s8, 0x7f
; SDAG-NEXT: v_cndmask_b32_e64 v16, v2, 0, s[4:5]
-; SDAG-NEXT: s_and_b64 s[10:11], s[10:11], s[6:7]
+; SDAG-NEXT: s_and_b64 s[8:9], s[8:9], s[6:7]
; SDAG-NEXT: v_cndmask_b32_e64 v32, v1, 0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v33, v0, 0, s[4:5]
-; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[10:11]
+; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[8:9]
; SDAG-NEXT: s_cbranch_execz .LBB5_6
; SDAG-NEXT: ; %bb.1: ; %udiv-bb15
; SDAG-NEXT: v_add_i32_e32 v30, vcc, 1, v18
; SDAG-NEXT: v_addc_u32_e32 v31, vcc, 0, v19, vcc
; SDAG-NEXT: v_addc_u32_e32 v32, vcc, 0, v20, vcc
; SDAG-NEXT: v_addc_u32_e64 v33, s[4:5], 0, v21, vcc
-; SDAG-NEXT: v_sub_i32_e32 v21, vcc, s8, v18
+; SDAG-NEXT: v_sub_i32_e32 v21, vcc, s10, v18
; SDAG-NEXT: v_sub_i32_e32 v19, vcc, 64, v21
; SDAG-NEXT: v_lshl_b64 v[16:17], v[2:3], v21
; SDAG-NEXT: v_lshr_b64 v[19:20], v[0:1], v19
-; SDAG-NEXT: s_xor_b64 s[8:9], s[4:5], -1
+; SDAG-NEXT: s_xor_b64 s[10:11], s[4:5], -1
; SDAG-NEXT: v_or_b32_e32 v19, v16, v19
; SDAG-NEXT: v_sub_i32_e32 v16, vcc, 63, v18
; SDAG-NEXT: v_or_b32_e32 v20, v17, v20
@@ -2591,14 +2591,15 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_lshl_b64 v[18:19], v[0:1], v21
; SDAG-NEXT: v_cndmask_b32_e32 v17, v17, v20, vcc
; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v21
+; SDAG-NEXT: s_mov_b64 s[8:9], 0
; SDAG-NEXT: v_cndmask_b32_e64 v17, v17, v3, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v16, v16, v2, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v19, 0, v19, vcc
; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: v_mov_b32_e32 v21, 0
; SDAG-NEXT: v_cndmask_b32_e32 v18, 0, v18, vcc
-; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[10:11]
+; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
; SDAG-NEXT: s_cbranch_execz .LBB5_5
; SDAG-NEXT: ; %bb.2: ; %udiv-preheader4
; SDAG-NEXT: v_sub_i32_e32 v22, vcc, 64, v30
@@ -2623,7 +2624,6 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_addc_u32_e32 v37, vcc, -1, v11, vcc
; SDAG-NEXT: v_mov_b32_e32 v24, 0
; SDAG-NEXT: v_mov_b32_e32 v25, 0
-; SDAG-NEXT: s_mov_b64 s[4:5], 0
; SDAG-NEXT: v_mov_b32_e32 v28, 0
; SDAG-NEXT: v_mov_b32_e32 v29, 0
; SDAG-NEXT: v_mov_b32_e32 v21, 0
@@ -2664,15 +2664,15 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_and_b32_e32 v20, 1, v38
; SDAG-NEXT: v_or_b32_e32 v17, v25, v17
; SDAG-NEXT: v_or_b32_e32 v16, v24, v16
-; SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; SDAG-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SDAG-NEXT: v_mov_b32_e32 v29, v21
; SDAG-NEXT: v_mov_b32_e32 v28, v20
-; SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; SDAG-NEXT: s_andn2_b64 exec, exec, s[8:9]
; SDAG-NEXT: s_cbranch_execnz .LBB5_3
; SDAG-NEXT: ; %bb.4: ; %Flow13
-; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
-; SDAG-NEXT: .LBB5_5: ; %Flow14
; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
+; SDAG-NEXT: .LBB5_5: ; %Flow14
+; SDAG-NEXT: s_or_b64 exec, exec, s[10:11]
; SDAG-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
; SDAG-NEXT: v_lshrrev_b32_e32 v22, 31, v19
; SDAG-NEXT: v_lshl_b64 v[18:19], v[18:19], 1
@@ -2712,7 +2712,7 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_add_i32_e32 v21, vcc, 64, v21
; SDAG-NEXT: v_addc_u32_e64 v22, s[6:7], 0, 0, vcc
; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; SDAG-NEXT: s_movk_i32 s8, 0x7f
+; SDAG-NEXT: s_movk_i32 s10, 0x7f
; SDAG-NEXT: v_cndmask_b32_e32 v19, v21, v19, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v22, v22, 0, vcc
; SDAG-NEXT: v_sub_i32_e32 v18, vcc, v18, v19
@@ -2746,7 +2746,7 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_addc_u32_e32 v35, vcc, 0, v19, vcc
; SDAG-NEXT: v_addc_u32_e32 v36, vcc, 0, v22, vcc
; SDAG-NEXT: v_addc_u32_e64 v37, s[4:5], 0, v23, vcc
-; SDAG-NEXT: v_sub_i32_e32 v23, vcc, s8, v18
+; SDAG-NEXT: v_sub_i32_e32 v23, vcc, s10, v18
; SDAG-NEXT: v_sub_i32_e32 v21, vcc, 64, v23
; SDAG-NEXT: v_lshl_b64 v[19:20], v[6:7], v23
; SDAG-NEXT: v_lshr_b64 v[21:22], v[4:5], v21
@@ -2755,19 +2755,20 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_or_b32_e32 v22, v19, v21
; SDAG-NEXT: v_lshl_b64 v[18:19], v[4:5], v18
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v23
-; SDAG-NEXT: s_xor_b64 s[8:9], s[4:5], -1
+; SDAG-NEXT: s_xor_b64 s[10:11], s[4:5], -1
; SDAG-NEXT: v_cndmask_b32_e32 v19, v19, v20, vcc
; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v23
; SDAG-NEXT: v_cndmask_b32_e64 v21, v19, v7, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v20, v18, v22, vcc
; SDAG-NEXT: v_lshl_b64 v[18:19], v[4:5], v23
+; SDAG-NEXT: s_mov_b64 s[8:9], 0
; SDAG-NEXT: v_cndmask_b32_e64 v20, v20, v6, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v19, 0, v19, vcc
; SDAG-NEXT: v_mov_b32_e32 v22, 0
; SDAG-NEXT: v_mov_b32_e32 v23, 0
; SDAG-NEXT: v_cndmask_b32_e32 v18, 0, v18, vcc
-; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[10:11]
+; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
; SDAG-NEXT: s_cbranch_execz .LBB5_11
; SDAG-NEXT: ; %bb.8: ; %udiv-preheader
; SDAG-NEXT: v_sub_i32_e32 v24, vcc, 64, v34
@@ -2792,7 +2793,6 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_addc_u32_e32 v49, vcc, -1, v15, vcc
; SDAG-NEXT: v_mov_b32_e32 v24, 0
; SDAG-NEXT: v_mov_b32_e32 v25, 0
-; SDAG-NEXT: s_mov_b64 s[10:11], 0
; SDAG-NEXT: v_mov_b32_e32 v28, 0
; SDAG-NEXT: v_mov_b32_e32 v29, 0
; SDAG-NEXT: v_mov_b32_e32 v23, 0
@@ -2832,16 +2832,16 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_subb_u32_e32 v30, vcc, v30, v51, vcc
; SDAG-NEXT: v_or_b32_e32 v21, v25, v21
; SDAG-NEXT: v_or_b32_e32 v20, v24, v20
-; SDAG-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11]
+; SDAG-NEXT: s_or_b64 s[8:9], s[4:5], s[8:9]
; SDAG-NEXT: v_mov_b32_e32 v29, v23
; SDAG-NEXT: v_mov_b32_e32 v28, v22
; SDAG-NEXT: v_subb_u32_e32 v31, vcc, v31, v50, vcc
-; SDAG-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; SDAG-NEXT: s_andn2_b64 exec, exec, s[8:9]
; SDAG-NEXT: s_cbranch_execnz .LBB5_9
; SDAG-NEXT: ; %bb.10: ; %Flow
-; SDAG-NEXT: s_or_b64 exec, exec, s[10:11]
-; SDAG-NEXT: .LBB5_11: ; %Flow11
; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
+; SDAG-NEXT: .LBB5_11: ; %Flow11
+; SDAG-NEXT: s_or_b64 exec, exec, s[10:11]
; SDAG-NEXT: v_lshl_b64 v[20:21], v[20:21], 1
; SDAG-NEXT: v_lshrrev_b32_e32 v24, 31, v19
; SDAG-NEXT: v_lshl_b64 v[18:19], v[18:19], 1
@@ -2949,29 +2949,29 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e64 v17, 0, 1, s[8:9]
; GISEL-NEXT: v_cmp_eq_u64_e64 s[8:9], 0, v[20:21]
+; GISEL-NEXT: s_movk_i32 s10, 0x7f
; GISEL-NEXT: v_cndmask_b32_e64 v16, v17, v16, s[8:9]
; GISEL-NEXT: v_and_b32_e32 v16, 1, v16
; GISEL-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v16
; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[10:11], s[4:5], -1
+; GISEL-NEXT: s_xor_b64 s[8:9], s[4:5], -1
; GISEL-NEXT: v_cndmask_b32_e64 v17, v3, 0, s[4:5]
-; GISEL-NEXT: s_movk_i32 s8, 0x7f
; GISEL-NEXT: v_cndmask_b32_e64 v16, v2, 0, s[4:5]
-; GISEL-NEXT: s_and_b64 s[10:11], s[10:11], s[6:7]
+; GISEL-NEXT: s_and_b64 s[8:9], s[8:9], s[6:7]
; GISEL-NEXT: v_cndmask_b32_e64 v32, v1, 0, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e64 v33, v0, 0, s[4:5]
-; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[10:11]
+; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[8:9]
; GISEL-NEXT: s_cbranch_execz .LBB5_6
; GISEL-NEXT: ; %bb.1: ; %udiv-bb15
; GISEL-NEXT: v_add_i32_e32 v30, vcc, 1, v18
; GISEL-NEXT: v_addc_u32_e32 v31, vcc, 0, v19, vcc
; GISEL-NEXT: v_addc_u32_e32 v32, vcc, 0, v20, vcc
; GISEL-NEXT: v_addc_u32_e64 v33, s[4:5], 0, v21, vcc
-; GISEL-NEXT: v_sub_i32_e32 v21, vcc, s8, v18
+; GISEL-NEXT: v_sub_i32_e32 v21, vcc, s10, v18
; GISEL-NEXT: v_sub_i32_e32 v19, vcc, 64, v21
; GISEL-NEXT: v_lshl_b64 v[16:17], v[2:3], v21
; GISEL-NEXT: v_lshr_b64 v[19:20], v[0:1], v19
-; GISEL-NEXT: s_xor_b64 s[8:9], s[4:5], -1
+; GISEL-NEXT: s_xor_b64 s[10:11], s[4:5], -1
; GISEL-NEXT: v_or_b32_e32 v19, v16, v19
; GISEL-NEXT: v_sub_i32_e32 v16, vcc, 63, v18
; GISEL-NEXT: v_or_b32_e32 v20, v17, v20
@@ -2981,14 +2981,15 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_lshl_b64 v[18:19], v[0:1], v21
; GISEL-NEXT: v_cndmask_b32_e32 v17, v17, v20, vcc
; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v21
+; GISEL-NEXT: s_mov_b64 s[8:9], 0
; GISEL-NEXT: v_cndmask_b32_e64 v17, v17, v3, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e64 v16, v16, v2, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e32 v19, 0, v19, vcc
; GISEL-NEXT: v_mov_b32_e32 v20, 0
; GISEL-NEXT: v_mov_b32_e32 v21, 0
; GISEL-NEXT: v_cndmask_b32_e32 v18, 0, v18, vcc
-; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[10:11]
+; GISEL-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
; GISEL-NEXT: s_cbranch_execz .LBB5_5
; GISEL-NEXT: ; %bb.2: ; %udiv-preheader4
; GISEL-NEXT: v_sub_i32_e32 v22, vcc, 64, v30
@@ -3013,7 +3014,6 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_addc_u32_e32 v37, vcc, -1, v11, vcc
; GISEL-NEXT: v_mov_b32_e32 v24, 0
; GISEL-NEXT: v_mov_b32_e32 v25, 0
-; GISEL-NEXT: s_mov_b64 s[4:5], 0
; GISEL-NEXT: v_mov_b32_e32 v28, 0
; GISEL-NEXT: v_mov_b32_e32 v29, 0
; GISEL-NEXT: v_mov_b32_e32 v21, 0
@@ -3054,15 +3054,15 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_and_b32_e32 v20, 1, v38
; GISEL-NEXT: v_or_b32_e32 v17, v25, v17
; GISEL-NEXT: v_or_b32_e32 v16, v24, v16
-; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GISEL-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GISEL-NEXT: v_mov_b32_e32 v29, v21
; GISEL-NEXT: v_mov_b32_e32 v28, v20
-; GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GISEL-NEXT: s_cbranch_execnz .LBB5_3
; GISEL-NEXT: ; %bb.4: ; %Flow13
-; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
-; GISEL-NEXT: .LBB5_5: ; %Flow14
; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
+; GISEL-NEXT: .LBB5_5: ; %Flow14
+; GISEL-NEXT: s_or_b64 exec, exec, s[10:11]
; GISEL-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
; GISEL-NEXT: v_lshrrev_b32_e32 v22, 31, v19
; GISEL-NEXT: v_lshl_b64 v[18:19], v[18:19], 1
@@ -3102,7 +3102,7 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_add_i32_e32 v21, vcc, 64, v21
; GISEL-NEXT: v_addc_u32_e64 v22, s[6:7], 0, 0, vcc
; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GISEL-NEXT: s_movk_i32 s8, 0x7f
+; GISEL-NEXT: s_movk_i32 s10, 0x7f
; GISEL-NEXT: v_cndmask_b32_e32 v19, v21, v19, vcc
; GISEL-NEXT: v_cndmask_b32_e64 v22, v22, 0, vcc
; GISEL-NEXT: v_sub_i32_e32 v18, vcc, v18, v19
@@ -3136,7 +3136,7 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_addc_u32_e32 v35, vcc, 0, v19, vcc
; GISEL-NEXT: v_addc_u32_e32 v36, vcc, 0, v22, vcc
; GISEL-NEXT: v_addc_u32_e64 v37, s[4:5], 0, v23, vcc
-; GISEL-NEXT: v_sub_i32_e32 v23, vcc, s8, v18
+; GISEL-NEXT: v_sub_i32_e32 v23, vcc, s10, v18
; GISEL-NEXT: v_sub_i32_e32 v21, vcc, 64, v23
; GISEL-NEXT: v_lshl_b64 v[19:20], v[6:7], v23
; GISEL-NEXT: v_lshr_b64 v[21:22], v[4:5], v21
@@ -3145,19 +3145,20 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_or_b32_e32 v22, v19, v21
; GISEL-NEXT: v_lshl_b64 v[18:19], v[4:5], v18
; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v23
-; GISEL-NEXT: s_xor_b64 s[8:9], s[4:5], -1
+; GISEL-NEXT: s_xor_b64 s[10:11], s[4:5], -1
; GISEL-NEXT: v_cndmask_b32_e32 v19, v19, v20, vcc
; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v23
; GISEL-NEXT: v_cndmask_b32_e64 v21, v19, v7, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e32 v20, v18, v22, vcc
; GISEL-NEXT: v_lshl_b64 v[18:19], v[4:5], v23
+; GISEL-NEXT: s_mov_b64 s[8:9], 0
; GISEL-NEXT: v_cndmask_b32_e64 v20, v20, v6, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e32 v19, 0, v19, vcc
; GISEL-NEXT: v_mov_b32_e32 v22, 0
; GISEL-NEXT: v_mov_b32_e32 v23, 0
; GISEL-NEXT: v_cndmask_b32_e32 v18, 0, v18, vcc
-; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[10:11]
+; GISEL-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
; GISEL-NEXT: s_cbranch_execz .LBB5_11
; GISEL-NEXT: ; %bb.8: ; %udiv-preheader
; GISEL-NEXT: v_sub_i32_e32 v24, vcc, 64, v34
@@ -3182,7 +3183,6 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_addc_u32_e32 v49, vcc, -1, v15, vcc
; GISEL-NEXT: v_mov_b32_e32 v24, 0
; GISEL-NEXT: v_mov_b32_e32 v25, 0
-; GISEL-NEXT: s_mov_b64 s[10:11], 0
; GISEL-NEXT: v_mov_b32_e32 v28, 0
; GISEL-NEXT: v_mov_b32_e32 v29, 0
; GISEL-NEXT: v_mov_b32_e32 v23, 0
@@ -3222,16 +3222,16 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_subb_u32_e32 v30, vcc, v30, v51, vcc
; GISEL-NEXT: v_or_b32_e32 v21, v25, v21
; GISEL-NEXT: v_or_b32_e32 v20, v24, v20
-; GISEL-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11]
+; GISEL-NEXT: s_or_b64 s[8:9], s[4:5], s[8:9]
; GISEL-NEXT: v_mov_b32_e32 v29, v23
; GISEL-NEXT: v_mov_b32_e32 v28, v22
; GISEL-NEXT: v_subb_u32_e32 v31, vcc, v31, v50, vcc
-; GISEL-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GISEL-NEXT: s_cbranch_execnz .LBB5_9
; GISEL-NEXT: ; %bb.10: ; %Flow
-; GISEL-NEXT: s_or_b64 exec, exec, s[10:11]
-; GISEL-NEXT: .LBB5_11: ; %Flow11
; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
+; GISEL-NEXT: .LBB5_11: ; %Flow11
+; GISEL-NEXT: s_or_b64 exec, exec, s[10:11]
; GISEL-NEXT: v_lshl_b64 v[20:21], v[20:21], 1
; GISEL-NEXT: v_lshrrev_b32_e32 v24, 31, v19
; GISEL-NEXT: v_lshl_b64 v[18:19], v[18:19], 1
diff --git a/llvm/test/CodeGen/AMDGPU/diverge-switch-default.ll b/llvm/test/CodeGen/AMDGPU/diverge-switch-default.ll
index f100478b00c0d..87bceb0dc9d37 100644
--- a/llvm/test/CodeGen/AMDGPU/diverge-switch-default.ll
+++ b/llvm/test/CodeGen/AMDGPU/diverge-switch-default.ll
@@ -32,13 +32,13 @@ sw.epilog:
; is sensitive to optimizations; so we just ensure that the relevant
; operations in the block body are indeed in the same block.
-; CHECK: [[PHI:%[a-zA-Z0-9._]+]] = phi i64
+; CHECK: [[PHI:%[a-zA-Z0-9._]+]] = phi i1
; CHECK-NOT: {{ br }}
; CHECK: load i8
; CHECK-NOT: {{ br }}
; CHECK: [[ICMP:%[a-zA-Z0-9._]+]] = icmp eq
-; CHECK: [[IF:%[a-zA-Z0-9._]+]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[ICMP]], i64 [[PHI]])
-; CHECK: [[LOOP:%[a-zA-Z0-9._]+]] = call i1 @llvm.amdgcn.loop.i64(i64 [[IF]])
+; CHECK: [[IF:%[a-zA-Z0-9._]+]] = call i1 @llvm.amdgcn.if.break(i1 [[ICMP]], i1 [[PHI]])
+; CHECK: [[LOOP:%[a-zA-Z0-9._]+]] = call i1 @llvm.amdgcn.loop(i1 [[IF]])
; CHECK: br i1 [[LOOP]]
sw.while:
diff --git a/llvm/test/CodeGen/AMDGPU/divergent-branch-uniform-condition.ll b/llvm/test/CodeGen/AMDGPU/divergent-branch-uniform-condition.ll
index 68abe6a409e66..aa636dcb963a6 100644
--- a/llvm/test/CodeGen/AMDGPU/divergent-branch-uniform-condition.ll
+++ b/llvm/test/CodeGen/AMDGPU/divergent-branch-uniform-condition.ll
@@ -90,10 +90,10 @@ Flow1: ; preds = %endif2, %endif1
; UNIFORM: if1:
; CONTROLFLOW-LABEL: Flow2:
-; CONTROLFLOW-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 %{{.*}})
-; CONTROLFLOW-NEXT: [[IF:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %{{.*}})
-; CONTROLFLOW-NEXT: [[COND:%.*]] = extractvalue { i1, i64 } [[IF]], 0
-; CONTROLFLOW-NEXT: %{{.*}} = extractvalue { i1, i64 } [[IF]], 1
+; CONTROLFLOW-NEXT: call void @llvm.amdgcn.end.cf(i1 %{{.*}})
+; CONTROLFLOW-NEXT: [[IF:%.*]] = call { i1, i1 } @llvm.amdgcn.if(i1 %{{.*}})
+; CONTROLFLOW-NEXT: [[COND:%.*]] = extractvalue { i1, i1 } [[IF]], 0
+; CONTROLFLOW-NEXT: %{{.*}} = extractvalue { i1, i1 } [[IF]], 1
; CONTROLFLOW-NEXT: br i1 [[COND]], label %if1, label %endloop
Flow2: ; preds = %Flow
diff --git a/llvm/test/CodeGen/AMDGPU/divergent-loop-i1-lane-mask.ll b/llvm/test/CodeGen/AMDGPU/divergent-loop-i1-lane-mask.ll
index 5864194a7c8a1..d1081c9914f32 100644
--- a/llvm/test/CodeGen/AMDGPU/divergent-loop-i1-lane-mask.ll
+++ b/llvm/test/CodeGen/AMDGPU/divergent-loop-i1-lane-mask.ll
@@ -34,18 +34,22 @@ define amdgpu_kernel void @divergent_loop(ptr addrspace(1) %p) {
;
; GFX9-GISEL-LABEL: divergent_loop:
; GFX9-GISEL: ; %bb.0: ; %entry
-; GFX9-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX9-GISEL-NEXT: s_mov_b32 s6, -1
; GFX9-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-GISEL-NEXT: ; implicit-def: $sgpr2_sgpr3
; GFX9-GISEL-NEXT: .LBB0_1: ; %loop
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-GISEL-NEXT: s_add_i32 s2, s2, 1
-; GFX9-GISEL-NEXT: v_cmp_ge_i32_e32 vcc, s2, v0
+; GFX9-GISEL-NEXT: s_add_i32 s6, s6, 1
+; GFX9-GISEL-NEXT: v_cmp_ge_i32_e32 vcc, s6, v0
; GFX9-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-GISEL-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
+; GFX9-GISEL-NEXT: s_and_b64 s[8:9], exec, s[0:1]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-GISEL-NEXT: s_or_b64 s[2:3], s[2:3], s[8:9]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB0_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %exit
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -74,18 +78,22 @@ define amdgpu_kernel void @divergent_loop(ptr addrspace(1) %p) {
;
; GFX10-GISEL-LABEL: divergent_loop:
; GFX10-GISEL: ; %bb.0: ; %entry
-; GFX10-GISEL-NEXT: s_mov_b32 s1, -1
-; GFX10-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX10-GISEL-NEXT: s_mov_b32 s0, -1
+; GFX10-GISEL-NEXT: s_mov_b32 s1, 0
+; GFX10-GISEL-NEXT: ; implicit-def: $sgpr2
; GFX10-GISEL-NEXT: .LBB0_1: ; %loop
; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-GISEL-NEXT: s_add_i32 s1, s1, 1
-; GFX10-GISEL-NEXT: v_cmp_ge_i32_e32 vcc_lo, s1, v0
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s0
+; GFX10-GISEL-NEXT: s_add_i32 s0, s0, 1
+; GFX10-GISEL-NEXT: v_cmp_ge_i32_e32 vcc_lo, s0, v0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX10-GISEL-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX10-GISEL-NEXT: s_andn2_b32 s2, s2, exec_lo
+; GFX10-GISEL-NEXT: s_and_b32 s3, exec_lo, s1
+; GFX10-GISEL-NEXT: s_or_b32 s2, s2, s3
+; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s1
; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB0_1
; GFX10-GISEL-NEXT: ; %bb.2: ; %exit
-; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
index e2614a08bbb16..f5fabf04f038f 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
@@ -13386,28 +13386,60 @@ define float @flat_atomic_fmax_f32_saddr_rtn(ptr inreg %ptr, float %data) {
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX950-LABEL: flat_atomic_fmax_f32_saddr_rtn:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_max_f32_e32 v4, v0, v0
-; GFX950-NEXT: .LBB118_1: ; %atomicrmw.start
-; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX950-NEXT: v_max_f32_e32 v0, v0, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
-; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX950-NEXT: s_cbranch_execnz .LBB118_1
-; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX950-NEXT: s_setpc_b64 s[30:31]
+; GFX950-SDAG-LABEL: flat_atomic_fmax_f32_saddr_rtn:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-SDAG-NEXT: flat_load_dword v1, v[2:3] offset:40
+; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-SDAG-NEXT: v_max_f32_e32 v4, v0, v0
+; GFX950-SDAG-NEXT: .LBB118_1: ; %atomicrmw.start
+; GFX950-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX950-SDAG-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX950-SDAG-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB118_1
+; GFX950-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: flat_atomic_fmax_f32_saddr_rtn:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-GISEL-NEXT: flat_load_dword v0, v[2:3] offset:40
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-GISEL-NEXT: v_max_f32_e32 v1, v1, v1
+; GFX950-GISEL-NEXT: ; implicit-def: $sgpr2_sgpr3
+; GFX950-GISEL-NEXT: ; implicit-def: $sgpr4_sgpr5
+; GFX950-GISEL-NEXT: .LBB118_1: ; %atomicrmw.start
+; GFX950-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v0
+; GFX950-GISEL-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX950-GISEL-NEXT: v_max_f32_e32 v4, v0, v1
+; GFX950-GISEL-NEXT: flat_atomic_cmpswap v0, v[2:3], v[4:5] offset:40 sc0
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GFX950-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX950-GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; GFX950-GISEL-NEXT: s_and_b64 s[6:7], exec, s[0:1]
+; GFX950-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
+; GFX950-GISEL-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
+; GFX950-GISEL-NEXT: s_and_b64 s[6:7], exec, s[4:5]
+; GFX950-GISEL-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
+; GFX950-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB118_1
+; GFX950-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 10
%result = atomicrmw fmax ptr %gep.0, float %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0
ret float %result
@@ -13424,28 +13456,59 @@ define void @flat_atomic_fmax_f32_saddr_nortn(ptr inreg %ptr, float %data) {
; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX950-LABEL: flat_atomic_fmax_f32_saddr_nortn:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_max_f32_e32 v4, v0, v0
-; GFX950-NEXT: .LBB119_1: ; %atomicrmw.start
-; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX950-NEXT: v_max_f32_e32 v0, v0, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
-; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX950-NEXT: s_cbranch_execnz .LBB119_1
-; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX950-NEXT: s_setpc_b64 s[30:31]
+; GFX950-SDAG-LABEL: flat_atomic_fmax_f32_saddr_nortn:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-SDAG-NEXT: flat_load_dword v1, v[2:3] offset:40
+; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-SDAG-NEXT: v_max_f32_e32 v4, v0, v0
+; GFX950-SDAG-NEXT: .LBB119_1: ; %atomicrmw.start
+; GFX950-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX950-SDAG-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX950-SDAG-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB119_1
+; GFX950-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: flat_atomic_fmax_f32_saddr_nortn:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: flat_load_dword v1, v[2:3] offset:40
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-GISEL-NEXT: v_max_f32_e32 v4, v0, v0
+; GFX950-GISEL-NEXT: ; implicit-def: $sgpr2_sgpr3
+; GFX950-GISEL-NEXT: ; implicit-def: $sgpr4_sgpr5
+; GFX950-GISEL-NEXT: .LBB119_1: ; %atomicrmw.start
+; GFX950-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX950-GISEL-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX950-GISEL-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX950-GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; GFX950-GISEL-NEXT: s_and_b64 s[6:7], exec, s[0:1]
+; GFX950-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
+; GFX950-GISEL-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
+; GFX950-GISEL-NEXT: s_and_b64 s[6:7], exec, s[4:5]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-GISEL-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
+; GFX950-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB119_1
+; GFX950-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 10
%unused = atomicrmw fmax ptr %gep.0, float %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0
ret void
@@ -13462,28 +13525,60 @@ define float @flat_atomic_fmin_f32_saddr_rtn(ptr inreg %ptr, float %data) {
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX950-LABEL: flat_atomic_fmin_f32_saddr_rtn:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_max_f32_e32 v4, v0, v0
-; GFX950-NEXT: .LBB120_1: ; %atomicrmw.start
-; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX950-NEXT: v_min_f32_e32 v0, v0, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
-; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX950-NEXT: s_cbranch_execnz .LBB120_1
-; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX950-NEXT: s_setpc_b64 s[30:31]
+; GFX950-SDAG-LABEL: flat_atomic_fmin_f32_saddr_rtn:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-SDAG-NEXT: flat_load_dword v1, v[2:3] offset:40
+; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-SDAG-NEXT: v_max_f32_e32 v4, v0, v0
+; GFX950-SDAG-NEXT: .LBB120_1: ; %atomicrmw.start
+; GFX950-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX950-SDAG-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX950-SDAG-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB120_1
+; GFX950-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: flat_atomic_fmin_f32_saddr_rtn:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-GISEL-NEXT: flat_load_dword v0, v[2:3] offset:40
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-GISEL-NEXT: v_max_f32_e32 v1, v1, v1
+; GFX950-GISEL-NEXT: ; implicit-def: $sgpr2_sgpr3
+; GFX950-GISEL-NEXT: ; implicit-def: $sgpr4_sgpr5
+; GFX950-GISEL-NEXT: .LBB120_1: ; %atomicrmw.start
+; GFX950-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v0
+; GFX950-GISEL-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX950-GISEL-NEXT: v_min_f32_e32 v4, v0, v1
+; GFX950-GISEL-NEXT: flat_atomic_cmpswap v0, v[2:3], v[4:5] offset:40 sc0
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GFX950-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX950-GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; GFX950-GISEL-NEXT: s_and_b64 s[6:7], exec, s[0:1]
+; GFX950-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
+; GFX950-GISEL-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
+; GFX950-GISEL-NEXT: s_and_b64 s[6:7], exec, s[4:5]
+; GFX950-GISEL-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
+; GFX950-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB120_1
+; GFX950-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 10
%result = atomicrmw fmin ptr %gep.0, float %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0
ret float %result
@@ -13500,28 +13595,59 @@ define void @flat_atomic_fmin_f32_saddr_nortn(ptr inreg %ptr, float %data) {
; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX950-LABEL: flat_atomic_fmin_f32_saddr_nortn:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_max_f32_e32 v4, v0, v0
-; GFX950-NEXT: .LBB121_1: ; %atomicrmw.start
-; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX950-NEXT: v_min_f32_e32 v0, v0, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
-; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX950-NEXT: s_cbranch_execnz .LBB121_1
-; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX950-NEXT: s_setpc_b64 s[30:31]
+; GFX950-SDAG-LABEL: flat_atomic_fmin_f32_saddr_nortn:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-SDAG-NEXT: flat_load_dword v1, v[2:3] offset:40
+; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-SDAG-NEXT: v_max_f32_e32 v4, v0, v0
+; GFX950-SDAG-NEXT: .LBB121_1: ; %atomicrmw.start
+; GFX950-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX950-SDAG-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX950-SDAG-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB121_1
+; GFX950-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: flat_atomic_fmin_f32_saddr_nortn:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: flat_load_dword v1, v[2:3] offset:40
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-GISEL-NEXT: v_max_f32_e32 v4, v0, v0
+; GFX950-GISEL-NEXT: ; implicit-def: $sgpr2_sgpr3
+; GFX950-GISEL-NEXT: ; implicit-def: $sgpr4_sgpr5
+; GFX950-GISEL-NEXT: .LBB121_1: ; %atomicrmw.start
+; GFX950-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX950-GISEL-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX950-GISEL-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX950-GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; GFX950-GISEL-NEXT: s_and_b64 s[6:7], exec, s[0:1]
+; GFX950-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
+; GFX950-GISEL-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
+; GFX950-GISEL-NEXT: s_and_b64 s[6:7], exec, s[4:5]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-GISEL-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
+; GFX950-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB121_1
+; GFX950-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 10
%unused = atomicrmw fmin ptr %gep.0, float %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0
ret void
@@ -13592,234 +13718,512 @@ define void @flat_atomic_fadd_v2f16_saddr_nortn(ptr inreg %ptr, <2 x half> %data
}
define <2 x half> @flat_atomic_fmax_v2f16_saddr_rtn(ptr inreg %ptr, <2 x half> %data) {
-; GFX1250-LABEL: flat_atomic_fmax_v2f16_saddr_rtn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, 0
-; GFX1250-NEXT: s_mov_b32 s2, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_max_num_f16 v1, v1, v1
-; GFX1250-NEXT: flat_load_b32 v0, v2, s[0:1] offset:40
-; GFX1250-NEXT: .LBB124_1: ; %atomicrmw.start
-; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v5, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_max_num_f16 v0, v5, v5
-; GFX1250-NEXT: v_pk_max_num_f16 v4, v0, v1
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[4:5], s[0:1] offset:40 th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
-; GFX1250-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
-; GFX1250-NEXT: s_cbranch_execnz .LBB124_1
-; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s2
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-SDAG-LABEL: flat_atomic_fmax_v2f16_saddr_rtn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, 0
+; GFX1250-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_pk_max_num_f16 v1, v1, v1
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v2, s[0:1] offset:40
+; GFX1250-SDAG-NEXT: .LBB124_1: ; %atomicrmw.start
+; GFX1250-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v5, v0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_pk_max_num_f16 v0, v5, v5
+; GFX1250-SDAG-NEXT: v_pk_max_num_f16 v4, v0, v1
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[4:5], s[0:1] offset:40 th:TH_ATOMIC_RETURN
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
+; GFX1250-SDAG-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB124_1
+; GFX1250-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX950-LABEL: flat_atomic_fmax_v2f16_saddr_rtn:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_pk_max_f16 v4, v0, v0
-; GFX950-NEXT: .p2align 5, , 4
-; GFX950-NEXT: .LBB124_1: ; %atomicrmw.start
-; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v0, v1, v1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_max_f16 v0, v0, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
-; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX950-NEXT: s_cbranch_execnz .LBB124_1
-; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX950-NEXT: s_setpc_b64 s[30:31]
+; GFX1250-GISEL-LABEL: flat_atomic_fmax_v2f16_saddr_rtn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, 0
+; GFX1250-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX1250-GISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1250-GISEL-NEXT: ; implicit-def: $sgpr4
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_pk_max_num_f16 v1, v1, v1
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v2, s[0:1] offset:40
+; GFX1250-GISEL-NEXT: .LBB124_1: ; %atomicrmw.start
+; GFX1250-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_pk_max_num_f16 v0, v5, v5
+; GFX1250-GISEL-NEXT: v_pk_max_num_f16 v4, v0, v1
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[4:5], s[0:1] offset:40 th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
+; GFX1250-GISEL-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1250-GISEL-NEXT: s_and_not1_b32 s4, s4, exec_lo
+; GFX1250-GISEL-NEXT: s_and_b32 s5, exec_lo, s2
+; GFX1250-GISEL-NEXT: s_and_not1_b32 s3, s3, exec_lo
+; GFX1250-GISEL-NEXT: s_or_b32 s4, s4, s5
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_and_b32 s5, exec_lo, s4
+; GFX1250-GISEL-NEXT: s_or_b32 s3, s3, s5
+; GFX1250-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB124_1
+; GFX1250-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX950-SDAG-LABEL: flat_atomic_fmax_v2f16_saddr_rtn:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-SDAG-NEXT: flat_load_dword v1, v[2:3] offset:40
+; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-SDAG-NEXT: v_pk_max_f16 v4, v0, v0
+; GFX950-SDAG-NEXT: .p2align 5, , 4
+; GFX950-SDAG-NEXT: .LBB124_1: ; %atomicrmw.start
+; GFX950-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX950-SDAG-NEXT: s_nop 0
+; GFX950-SDAG-NEXT: v_pk_max_f16 v0, v0, v4
+; GFX950-SDAG-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB124_1
+; GFX950-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: flat_atomic_fmax_v2f16_saddr_rtn:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-GISEL-NEXT: flat_load_dword v0, v[2:3] offset:40
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-GISEL-NEXT: v_pk_max_f16 v1, v1, v1
+; GFX950-GISEL-NEXT: ; implicit-def: $sgpr2_sgpr3
+; GFX950-GISEL-NEXT: ; implicit-def: $sgpr4_sgpr5
+; GFX950-GISEL-NEXT: .LBB124_1: ; %atomicrmw.start
+; GFX950-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v0
+; GFX950-GISEL-NEXT: v_pk_max_f16 v0, v5, v5
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_pk_max_f16 v4, v0, v1
+; GFX950-GISEL-NEXT: flat_atomic_cmpswap v0, v[2:3], v[4:5] offset:40 sc0
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GFX950-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX950-GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; GFX950-GISEL-NEXT: s_and_b64 s[6:7], exec, s[0:1]
+; GFX950-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
+; GFX950-GISEL-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
+; GFX950-GISEL-NEXT: s_and_b64 s[6:7], exec, s[4:5]
+; GFX950-GISEL-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
+; GFX950-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB124_1
+; GFX950-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 10
%result = atomicrmw fmax ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %result
}
define void @flat_atomic_fmax_v2f16_saddr_nortn(ptr inreg %ptr, <2 x half> %data) {
-; GFX1250-LABEL: flat_atomic_fmax_v2f16_saddr_nortn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-NEXT: v_pk_max_num_f16 v3, v0, v0
-; GFX1250-NEXT: s_mov_b32 s2, 0
-; GFX1250-NEXT: flat_load_b32 v1, v2, s[0:1] offset:40
-; GFX1250-NEXT: .LBB125_1: ; %atomicrmw.start
-; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_pk_max_num_f16 v0, v1, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_max_num_f16 v0, v0, v3
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:40 th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
-; GFX1250-NEXT: v_mov_b32_e32 v1, v0
-; GFX1250-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
-; GFX1250-NEXT: s_cbranch_execnz .LBB125_1
-; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s2
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-SDAG-LABEL: flat_atomic_fmax_v2f16_saddr_nortn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-SDAG-NEXT: v_pk_max_num_f16 v3, v0, v0
+; GFX1250-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX1250-SDAG-NEXT: flat_load_b32 v1, v2, s[0:1] offset:40
+; GFX1250-SDAG-NEXT: .LBB125_1: ; %atomicrmw.start
+; GFX1250-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: v_pk_max_num_f16 v0, v1, v1
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_pk_max_num_f16 v0, v0, v3
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:40 th:TH_ATOMIC_RETURN
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX1250-SDAG-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB125_1
+; GFX1250-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX950-LABEL: flat_atomic_fmax_v2f16_saddr_nortn:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_pk_max_f16 v4, v0, v0
-; GFX950-NEXT: .p2align 5, , 4
-; GFX950-NEXT: .LBB125_1: ; %atomicrmw.start
-; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v0, v1, v1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_max_f16 v0, v0, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
-; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX950-NEXT: s_cbranch_execnz .LBB125_1
-; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX950-NEXT: s_setpc_b64 s[30:31]
+; GFX1250-GISEL-LABEL: flat_atomic_fmax_v2f16_saddr_nortn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-GISEL-NEXT: v_pk_max_num_f16 v3, v0, v0
+; GFX1250-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX1250-GISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1250-GISEL-NEXT: ; implicit-def: $sgpr4
+; GFX1250-GISEL-NEXT: flat_load_b32 v1, v2, s[0:1] offset:40
+; GFX1250-GISEL-NEXT: .LBB125_1: ; %atomicrmw.start
+; GFX1250-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_pk_max_num_f16 v0, v1, v1
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_pk_max_num_f16 v0, v0, v3
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:40 th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX1250-GISEL-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1250-GISEL-NEXT: s_and_not1_b32 s4, s4, exec_lo
+; GFX1250-GISEL-NEXT: s_and_b32 s5, exec_lo, s2
+; GFX1250-GISEL-NEXT: s_and_not1_b32 s3, s3, exec_lo
+; GFX1250-GISEL-NEXT: s_or_b32 s4, s4, s5
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_and_b32 s5, exec_lo, s4
+; GFX1250-GISEL-NEXT: s_or_b32 s3, s3, s5
+; GFX1250-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB125_1
+; GFX1250-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX950-SDAG-LABEL: flat_atomic_fmax_v2f16_saddr_nortn:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-SDAG-NEXT: flat_load_dword v1, v[2:3] offset:40
+; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-SDAG-NEXT: v_pk_max_f16 v4, v0, v0
+; GFX950-SDAG-NEXT: .p2align 5, , 4
+; GFX950-SDAG-NEXT: .LBB125_1: ; %atomicrmw.start
+; GFX950-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX950-SDAG-NEXT: s_nop 0
+; GFX950-SDAG-NEXT: v_pk_max_f16 v0, v0, v4
+; GFX950-SDAG-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB125_1
+; GFX950-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: flat_atomic_fmax_v2f16_saddr_nortn:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: flat_load_dword v1, v[2:3] offset:40
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-GISEL-NEXT: v_pk_max_f16 v4, v0, v0
+; GFX950-GISEL-NEXT: ; implicit-def: $sgpr2_sgpr3
+; GFX950-GISEL-NEXT: ; implicit-def: $sgpr4_sgpr5
+; GFX950-GISEL-NEXT: .p2align 5, , 4
+; GFX950-GISEL-NEXT: .LBB125_1: ; %atomicrmw.start
+; GFX950-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_pk_max_f16 v0, v0, v4
+; GFX950-GISEL-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX950-GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; GFX950-GISEL-NEXT: s_and_b64 s[6:7], exec, s[0:1]
+; GFX950-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
+; GFX950-GISEL-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
+; GFX950-GISEL-NEXT: s_and_b64 s[6:7], exec, s[4:5]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-GISEL-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
+; GFX950-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB125_1
+; GFX950-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 10
%unused = atomicrmw fmax ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0
ret void
}
-define <2 x half> @flat_atomic_fmin_v2f16_saddr_rtn(ptr inreg %ptr, <2 x half> %data) {
-; GFX1250-LABEL: flat_atomic_fmin_v2f16_saddr_rtn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, 0
-; GFX1250-NEXT: s_mov_b32 s2, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_max_num_f16 v1, v1, v1
-; GFX1250-NEXT: flat_load_b32 v0, v2, s[0:1] offset:40
-; GFX1250-NEXT: .LBB126_1: ; %atomicrmw.start
-; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v5, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_max_num_f16 v0, v5, v5
-; GFX1250-NEXT: v_pk_min_num_f16 v4, v0, v1
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[4:5], s[0:1] offset:40 th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
-; GFX1250-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
-; GFX1250-NEXT: s_cbranch_execnz .LBB126_1
-; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s2
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+define <2 x half> @flat_atomic_fmin_v2f16_saddr_rtn(ptr inreg %ptr, <2 x half> %data) {
+; GFX1250-SDAG-LABEL: flat_atomic_fmin_v2f16_saddr_rtn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, 0
+; GFX1250-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_pk_max_num_f16 v1, v1, v1
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v2, s[0:1] offset:40
+; GFX1250-SDAG-NEXT: .LBB126_1: ; %atomicrmw.start
+; GFX1250-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v5, v0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_pk_max_num_f16 v0, v5, v5
+; GFX1250-SDAG-NEXT: v_pk_min_num_f16 v4, v0, v1
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[4:5], s[0:1] offset:40 th:TH_ATOMIC_RETURN
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
+; GFX1250-SDAG-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB126_1
+; GFX1250-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX950-LABEL: flat_atomic_fmin_v2f16_saddr_rtn:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_pk_max_f16 v4, v0, v0
-; GFX950-NEXT: .p2align 5, , 4
-; GFX950-NEXT: .LBB126_1: ; %atomicrmw.start
-; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v0, v1, v1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_min_f16 v0, v0, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
-; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX950-NEXT: s_cbranch_execnz .LBB126_1
-; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX950-NEXT: s_setpc_b64 s[30:31]
+; GFX1250-GISEL-LABEL: flat_atomic_fmin_v2f16_saddr_rtn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, 0
+; GFX1250-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX1250-GISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1250-GISEL-NEXT: ; implicit-def: $sgpr4
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_pk_max_num_f16 v1, v1, v1
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v2, s[0:1] offset:40
+; GFX1250-GISEL-NEXT: .LBB126_1: ; %atomicrmw.start
+; GFX1250-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_pk_max_num_f16 v0, v5, v5
+; GFX1250-GISEL-NEXT: v_pk_min_num_f16 v4, v0, v1
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[4:5], s[0:1] offset:40 th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
+; GFX1250-GISEL-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1250-GISEL-NEXT: s_and_not1_b32 s4, s4, exec_lo
+; GFX1250-GISEL-NEXT: s_and_b32 s5, exec_lo, s2
+; GFX1250-GISEL-NEXT: s_and_not1_b32 s3, s3, exec_lo
+; GFX1250-GISEL-NEXT: s_or_b32 s4, s4, s5
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_and_b32 s5, exec_lo, s4
+; GFX1250-GISEL-NEXT: s_or_b32 s3, s3, s5
+; GFX1250-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB126_1
+; GFX1250-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX950-SDAG-LABEL: flat_atomic_fmin_v2f16_saddr_rtn:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-SDAG-NEXT: flat_load_dword v1, v[2:3] offset:40
+; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-SDAG-NEXT: v_pk_max_f16 v4, v0, v0
+; GFX950-SDAG-NEXT: .p2align 5, , 4
+; GFX950-SDAG-NEXT: .LBB126_1: ; %atomicrmw.start
+; GFX950-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX950-SDAG-NEXT: s_nop 0
+; GFX950-SDAG-NEXT: v_pk_min_f16 v0, v0, v4
+; GFX950-SDAG-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB126_1
+; GFX950-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: flat_atomic_fmin_v2f16_saddr_rtn:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-GISEL-NEXT: flat_load_dword v0, v[2:3] offset:40
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-GISEL-NEXT: v_pk_max_f16 v1, v1, v1
+; GFX950-GISEL-NEXT: ; implicit-def: $sgpr2_sgpr3
+; GFX950-GISEL-NEXT: ; implicit-def: $sgpr4_sgpr5
+; GFX950-GISEL-NEXT: .LBB126_1: ; %atomicrmw.start
+; GFX950-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v0
+; GFX950-GISEL-NEXT: v_pk_max_f16 v0, v5, v5
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_pk_min_f16 v4, v0, v1
+; GFX950-GISEL-NEXT: flat_atomic_cmpswap v0, v[2:3], v[4:5] offset:40 sc0
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GFX950-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX950-GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; GFX950-GISEL-NEXT: s_and_b64 s[6:7], exec, s[0:1]
+; GFX950-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
+; GFX950-GISEL-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
+; GFX950-GISEL-NEXT: s_and_b64 s[6:7], exec, s[4:5]
+; GFX950-GISEL-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
+; GFX950-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB126_1
+; GFX950-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 10
%result = atomicrmw fmin ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %result
}
define void @flat_atomic_fmin_v2f16_saddr_nortn(ptr inreg %ptr, <2 x half> %data) {
-; GFX1250-LABEL: flat_atomic_fmin_v2f16_saddr_nortn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-NEXT: v_pk_max_num_f16 v3, v0, v0
-; GFX1250-NEXT: s_mov_b32 s2, 0
-; GFX1250-NEXT: flat_load_b32 v1, v2, s[0:1] offset:40
-; GFX1250-NEXT: .LBB127_1: ; %atomicrmw.start
-; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_pk_max_num_f16 v0, v1, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_min_num_f16 v0, v0, v3
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:40 th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
-; GFX1250-NEXT: v_mov_b32_e32 v1, v0
-; GFX1250-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
-; GFX1250-NEXT: s_cbranch_execnz .LBB127_1
-; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s2
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-SDAG-LABEL: flat_atomic_fmin_v2f16_saddr_nortn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-SDAG-NEXT: v_pk_max_num_f16 v3, v0, v0
+; GFX1250-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX1250-SDAG-NEXT: flat_load_b32 v1, v2, s[0:1] offset:40
+; GFX1250-SDAG-NEXT: .LBB127_1: ; %atomicrmw.start
+; GFX1250-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: v_pk_max_num_f16 v0, v1, v1
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_pk_min_num_f16 v0, v0, v3
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:40 th:TH_ATOMIC_RETURN
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX1250-SDAG-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB127_1
+; GFX1250-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX950-LABEL: flat_atomic_fmin_v2f16_saddr_nortn:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_pk_max_f16 v4, v0, v0
-; GFX950-NEXT: .p2align 5, , 4
-; GFX950-NEXT: .LBB127_1: ; %atomicrmw.start
-; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v0, v1, v1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_min_f16 v0, v0, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
-; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX950-NEXT: s_cbranch_execnz .LBB127_1
-; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX950-NEXT: s_setpc_b64 s[30:31]
+; GFX1250-GISEL-LABEL: flat_atomic_fmin_v2f16_saddr_nortn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-GISEL-NEXT: v_pk_max_num_f16 v3, v0, v0
+; GFX1250-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX1250-GISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1250-GISEL-NEXT: ; implicit-def: $sgpr4
+; GFX1250-GISEL-NEXT: flat_load_b32 v1, v2, s[0:1] offset:40
+; GFX1250-GISEL-NEXT: .LBB127_1: ; %atomicrmw.start
+; GFX1250-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_pk_max_num_f16 v0, v1, v1
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_pk_min_num_f16 v0, v0, v3
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:40 th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX1250-GISEL-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1250-GISEL-NEXT: s_and_not1_b32 s4, s4, exec_lo
+; GFX1250-GISEL-NEXT: s_and_b32 s5, exec_lo, s2
+; GFX1250-GISEL-NEXT: s_and_not1_b32 s3, s3, exec_lo
+; GFX1250-GISEL-NEXT: s_or_b32 s4, s4, s5
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_and_b32 s5, exec_lo, s4
+; GFX1250-GISEL-NEXT: s_or_b32 s3, s3, s5
+; GFX1250-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB127_1
+; GFX1250-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX950-SDAG-LABEL: flat_atomic_fmin_v2f16_saddr_nortn:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-SDAG-NEXT: flat_load_dword v1, v[2:3] offset:40
+; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-SDAG-NEXT: v_pk_max_f16 v4, v0, v0
+; GFX950-SDAG-NEXT: .p2align 5, , 4
+; GFX950-SDAG-NEXT: .LBB127_1: ; %atomicrmw.start
+; GFX950-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX950-SDAG-NEXT: s_nop 0
+; GFX950-SDAG-NEXT: v_pk_min_f16 v0, v0, v4
+; GFX950-SDAG-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB127_1
+; GFX950-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: flat_atomic_fmin_v2f16_saddr_nortn:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: flat_load_dword v1, v[2:3] offset:40
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-GISEL-NEXT: v_pk_max_f16 v4, v0, v0
+; GFX950-GISEL-NEXT: ; implicit-def: $sgpr2_sgpr3
+; GFX950-GISEL-NEXT: ; implicit-def: $sgpr4_sgpr5
+; GFX950-GISEL-NEXT: .p2align 5, , 4
+; GFX950-GISEL-NEXT: .LBB127_1: ; %atomicrmw.start
+; GFX950-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_pk_min_f16 v0, v0, v4
+; GFX950-GISEL-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX950-GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; GFX950-GISEL-NEXT: s_and_b64 s[6:7], exec, s[0:1]
+; GFX950-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
+; GFX950-GISEL-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
+; GFX950-GISEL-NEXT: s_and_b64 s[6:7], exec, s[4:5]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-GISEL-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
+; GFX950-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB127_1
+; GFX950-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 10
%unused = atomicrmw fmin ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0
ret void
@@ -13910,32 +14314,69 @@ define void @flat_atomic_fadd_v2bf16_saddr_nortn(ptr inreg %ptr, <2 x bfloat> %d
}
define <2 x bfloat> @flat_atomic_fmax_v2bf16_saddr_rtn(ptr inreg %ptr, <2 x bfloat> %data) {
-; GFX1250-LABEL: flat_atomic_fmax_v2bf16_saddr_rtn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-NEXT: s_mov_b32 s2, 0
-; GFX1250-NEXT: flat_load_b32 v1, v2, s[0:1] offset:40
-; GFX1250-NEXT: .LBB130_1: ; %atomicrmw.start
-; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v5, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_max_num_bf16 v4, v5, v0
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: flat_atomic_cmpswap_b32 v1, v2, v[4:5], s[0:1] offset:40 th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
-; GFX1250-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
-; GFX1250-NEXT: s_cbranch_execnz .LBB130_1
-; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s2
-; GFX1250-NEXT: v_mov_b32_e32 v0, v1
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-SDAG-LABEL: flat_atomic_fmax_v2bf16_saddr_rtn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX1250-SDAG-NEXT: flat_load_b32 v1, v2, s[0:1] offset:40
+; GFX1250-SDAG-NEXT: .LBB130_1: ; %atomicrmw.start
+; GFX1250-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v5, v1
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_pk_max_num_bf16 v4, v5, v0
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_cmpswap_b32 v1, v2, v[4:5], s[0:1] offset:40 th:TH_ATOMIC_RETURN
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
+; GFX1250-SDAG-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB130_1
+; GFX1250-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: flat_atomic_fmax_v2bf16_saddr_rtn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, 0
+; GFX1250-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX1250-GISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1250-GISEL-NEXT: ; implicit-def: $sgpr4
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_pk_mul_bf16 v1, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v2, s[0:1] offset:40
+; GFX1250-GISEL-NEXT: .LBB130_1: ; %atomicrmw.start
+; GFX1250-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_pk_mul_bf16 v0, 1.0, v5 op_sel_hi:[0,1]
+; GFX1250-GISEL-NEXT: v_pk_max_num_bf16 v4, v0, v1
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[4:5], s[0:1] offset:40 th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
+; GFX1250-GISEL-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1250-GISEL-NEXT: s_and_not1_b32 s4, s4, exec_lo
+; GFX1250-GISEL-NEXT: s_and_b32 s5, exec_lo, s2
+; GFX1250-GISEL-NEXT: s_and_not1_b32 s3, s3, exec_lo
+; GFX1250-GISEL-NEXT: s_or_b32 s4, s4, s5
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_and_b32 s5, exec_lo, s4
+; GFX1250-GISEL-NEXT: s_or_b32 s3, s3, s5
+; GFX1250-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB130_1
+; GFX1250-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
;
; GFX950-LABEL: flat_atomic_fmax_v2bf16_saddr_rtn:
; GFX950: ; %bb.0:
@@ -13970,30 +14411,66 @@ define <2 x bfloat> @flat_atomic_fmax_v2bf16_saddr_rtn(ptr inreg %ptr, <2 x bflo
}
define void @flat_atomic_fmax_v2bf16_saddr_nortn(ptr inreg %ptr, <2 x bfloat> %data) {
-; GFX1250-LABEL: flat_atomic_fmax_v2bf16_saddr_nortn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NEXT: s_mov_b32 s2, 0
-; GFX1250-NEXT: flat_load_b32 v3, v1, s[0:1] offset:40
-; GFX1250-NEXT: .LBB131_1: ; %atomicrmw.start
-; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_pk_max_num_bf16 v2, v3, v0
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: flat_atomic_cmpswap_b32 v2, v1, v[2:3], s[0:1] offset:40 th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1250-NEXT: v_mov_b32_e32 v3, v2
-; GFX1250-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
-; GFX1250-NEXT: s_cbranch_execnz .LBB131_1
-; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s2
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-SDAG-LABEL: flat_atomic_fmax_v2bf16_saddr_nortn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX1250-SDAG-NEXT: flat_load_b32 v3, v1, s[0:1] offset:40
+; GFX1250-SDAG-NEXT: .LBB131_1: ; %atomicrmw.start
+; GFX1250-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: v_pk_max_num_bf16 v2, v3, v0
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_cmpswap_b32 v2, v1, v[2:3], s[0:1] offset:40 th:TH_ATOMIC_RETURN
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX1250-SDAG-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB131_1
+; GFX1250-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: flat_atomic_fmax_v2bf16_saddr_nortn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-GISEL-NEXT: v_pk_mul_bf16 v3, 1.0, v0 op_sel_hi:[0,1]
+; GFX1250-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX1250-GISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1250-GISEL-NEXT: ; implicit-def: $sgpr4
+; GFX1250-GISEL-NEXT: flat_load_b32 v1, v2, s[0:1] offset:40
+; GFX1250-GISEL-NEXT: .LBB131_1: ; %atomicrmw.start
+; GFX1250-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_pk_mul_bf16 v0, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_pk_max_num_bf16 v0, v0, v3
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:40 th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX1250-GISEL-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1250-GISEL-NEXT: s_and_not1_b32 s4, s4, exec_lo
+; GFX1250-GISEL-NEXT: s_and_b32 s5, exec_lo, s2
+; GFX1250-GISEL-NEXT: s_and_not1_b32 s3, s3, exec_lo
+; GFX1250-GISEL-NEXT: s_or_b32 s4, s4, s5
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_and_b32 s5, exec_lo, s4
+; GFX1250-GISEL-NEXT: s_or_b32 s3, s3, s5
+; GFX1250-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB131_1
+; GFX1250-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
;
; GFX950-LABEL: flat_atomic_fmax_v2bf16_saddr_nortn:
; GFX950: ; %bb.0:
@@ -14028,32 +14505,69 @@ define void @flat_atomic_fmax_v2bf16_saddr_nortn(ptr inreg %ptr, <2 x bfloat> %d
}
define <2 x bfloat> @flat_atomic_fmin_v2bf16_saddr_rtn(ptr inreg %ptr, <2 x bfloat> %data) {
-; GFX1250-LABEL: flat_atomic_fmin_v2bf16_saddr_rtn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-NEXT: s_mov_b32 s2, 0
-; GFX1250-NEXT: flat_load_b32 v1, v2, s[0:1] offset:40
-; GFX1250-NEXT: .LBB132_1: ; %atomicrmw.start
-; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v5, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_min_num_bf16 v4, v5, v0
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: flat_atomic_cmpswap_b32 v1, v2, v[4:5], s[0:1] offset:40 th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
-; GFX1250-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
-; GFX1250-NEXT: s_cbranch_execnz .LBB132_1
-; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s2
-; GFX1250-NEXT: v_mov_b32_e32 v0, v1
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-SDAG-LABEL: flat_atomic_fmin_v2bf16_saddr_rtn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX1250-SDAG-NEXT: flat_load_b32 v1, v2, s[0:1] offset:40
+; GFX1250-SDAG-NEXT: .LBB132_1: ; %atomicrmw.start
+; GFX1250-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v5, v1
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_pk_min_num_bf16 v4, v5, v0
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_cmpswap_b32 v1, v2, v[4:5], s[0:1] offset:40 th:TH_ATOMIC_RETURN
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
+; GFX1250-SDAG-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB132_1
+; GFX1250-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: flat_atomic_fmin_v2bf16_saddr_rtn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, 0
+; GFX1250-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX1250-GISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1250-GISEL-NEXT: ; implicit-def: $sgpr4
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_pk_mul_bf16 v1, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v2, s[0:1] offset:40
+; GFX1250-GISEL-NEXT: .LBB132_1: ; %atomicrmw.start
+; GFX1250-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_pk_mul_bf16 v0, 1.0, v5 op_sel_hi:[0,1]
+; GFX1250-GISEL-NEXT: v_pk_min_num_bf16 v4, v0, v1
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[4:5], s[0:1] offset:40 th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
+; GFX1250-GISEL-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1250-GISEL-NEXT: s_and_not1_b32 s4, s4, exec_lo
+; GFX1250-GISEL-NEXT: s_and_b32 s5, exec_lo, s2
+; GFX1250-GISEL-NEXT: s_and_not1_b32 s3, s3, exec_lo
+; GFX1250-GISEL-NEXT: s_or_b32 s4, s4, s5
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_and_b32 s5, exec_lo, s4
+; GFX1250-GISEL-NEXT: s_or_b32 s3, s3, s5
+; GFX1250-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB132_1
+; GFX1250-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
;
; GFX950-LABEL: flat_atomic_fmin_v2bf16_saddr_rtn:
; GFX950: ; %bb.0:
@@ -14088,30 +14602,66 @@ define <2 x bfloat> @flat_atomic_fmin_v2bf16_saddr_rtn(ptr inreg %ptr, <2 x bflo
}
define void @flat_atomic_fmin_v2bf16_saddr_nortn(ptr inreg %ptr, <2 x bfloat> %data) {
-; GFX1250-LABEL: flat_atomic_fmin_v2bf16_saddr_nortn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NEXT: s_mov_b32 s2, 0
-; GFX1250-NEXT: flat_load_b32 v3, v1, s[0:1] offset:40
-; GFX1250-NEXT: .LBB133_1: ; %atomicrmw.start
-; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_pk_min_num_bf16 v2, v3, v0
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: flat_atomic_cmpswap_b32 v2, v1, v[2:3], s[0:1] offset:40 th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1250-NEXT: v_mov_b32_e32 v3, v2
-; GFX1250-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
-; GFX1250-NEXT: s_cbranch_execnz .LBB133_1
-; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s2
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-SDAG-LABEL: flat_atomic_fmin_v2bf16_saddr_nortn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX1250-SDAG-NEXT: flat_load_b32 v3, v1, s[0:1] offset:40
+; GFX1250-SDAG-NEXT: .LBB133_1: ; %atomicrmw.start
+; GFX1250-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: v_pk_min_num_bf16 v2, v3, v0
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_cmpswap_b32 v2, v1, v[2:3], s[0:1] offset:40 th:TH_ATOMIC_RETURN
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX1250-SDAG-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB133_1
+; GFX1250-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: flat_atomic_fmin_v2bf16_saddr_nortn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-GISEL-NEXT: v_pk_mul_bf16 v3, 1.0, v0 op_sel_hi:[0,1]
+; GFX1250-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX1250-GISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1250-GISEL-NEXT: ; implicit-def: $sgpr4
+; GFX1250-GISEL-NEXT: flat_load_b32 v1, v2, s[0:1] offset:40
+; GFX1250-GISEL-NEXT: .LBB133_1: ; %atomicrmw.start
+; GFX1250-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_pk_mul_bf16 v0, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_pk_min_num_bf16 v0, v0, v3
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:40 th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX1250-GISEL-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1250-GISEL-NEXT: s_and_not1_b32 s4, s4, exec_lo
+; GFX1250-GISEL-NEXT: s_and_b32 s5, exec_lo, s2
+; GFX1250-GISEL-NEXT: s_and_not1_b32 s3, s3, exec_lo
+; GFX1250-GISEL-NEXT: s_or_b32 s4, s4, s5
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_and_b32 s5, exec_lo, s4
+; GFX1250-GISEL-NEXT: s_or_b32 s3, s3, s5
+; GFX1250-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB133_1
+; GFX1250-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
;
; GFX950-LABEL: flat_atomic_fmin_v2bf16_saddr_nortn:
; GFX950: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/irtranslator-whole-wave-functions.ll b/llvm/test/CodeGen/AMDGPU/irtranslator-whole-wave-functions.ll
index 2edc78915e0eb..97f51d459afee 100644
--- a/llvm/test/CodeGen/AMDGPU/irtranslator-whole-wave-functions.ll
+++ b/llvm/test/CodeGen/AMDGPU/irtranslator-whole-wave-functions.ll
@@ -47,7 +47,7 @@ define amdgpu_gfx_whole_wave i32 @multiple_blocks(i1 %active, i32 %a, i32 %b) {
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
; CHECK-NEXT: [[AMDGPU_WHOLE_WAVE_FUNC_SETUP:%[0-9]+]]:_(i1) = G_AMDGPU_WHOLE_WAVE_FUNC_SETUP
; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i1) = G_ICMP intpred(eq), [[COPY]](i32), [[COPY1]]
- ; CHECK-NEXT: [[INT:%[0-9]+]]:_(i1), [[INT1:%[0-9]+]]:_(i32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.if), [[ICMP]](i1)
+ ; CHECK-NEXT: [[INT:%[0-9]+]]:_(i1), [[INT1:%[0-9]+]]:_(i1) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.if), [[ICMP]](i1)
; CHECK-NEXT: G_BRCOND [[INT]](i1), %bb.2
; CHECK-NEXT: G_BR %bb.3
; CHECK-NEXT: {{ $}}
@@ -58,7 +58,7 @@ define amdgpu_gfx_whole_wave i32 @multiple_blocks(i1 %active, i32 %a, i32 %b) {
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3.if.end:
; CHECK-NEXT: [[PHI:%[0-9]+]]:_(i32) = G_PHI [[COPY1]](i32), %bb.1, [[ADD]](i32), %bb.2
- ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT1]](i32)
+ ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT1]](i1)
; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[AMDGPU_WHOLE_WAVE_FUNC_SETUP]](i1), [[COPY]], [[PHI]]
; CHECK-NEXT: $vgpr0 = COPY [[SELECT]](i32)
; CHECK-NEXT: G_AMDGPU_WHOLE_WAVE_FUNC_RETURN [[AMDGPU_WHOLE_WAVE_FUNC_SETUP]](i1), implicit $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/literal64.ll b/llvm/test/CodeGen/AMDGPU/literal64.ll
index 5c8d25fb0f021..059ca985665c2 100644
--- a/llvm/test/CodeGen/AMDGPU/literal64.ll
+++ b/llvm/test/CodeGen/AMDGPU/literal64.ll
@@ -130,30 +130,63 @@ define void @v_mov_b64_double(ptr addrspace(1) %ptr) {
; GFX1250-NEXT: global_atomic_add_f64 v[0:1], v[2:3], off scope:SCOPE_SYS
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX13-LABEL: v_mov_b64_double:
-; GFX13: ; %bb.0:
-; GFX13-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX13-NEXT: s_wait_expcnt 0x0
-; GFX13-NEXT: s_wait_samplecnt 0x0
-; GFX13-NEXT: s_wait_bvhcnt 0x0
-; GFX13-NEXT: s_wait_kmcnt 0x0
-; GFX13-NEXT: global_load_b64 v[4:5], v[0:1], off
-; GFX13-NEXT: s_mov_b32 s0, 0
-; GFX13-NEXT: .LBB6_1: ; %atomicrmw.start
-; GFX13-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX13-NEXT: s_wait_loadcnt 0x0
-; GFX13-NEXT: v_add_f64_e32 v[2:3], 0x4063233333333333, v[4:5]
-; GFX13-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX13-NEXT: s_wait_loadcnt 0x0
-; GFX13-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
-; GFX13-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
-; GFX13-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX13-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX13-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX13-NEXT: s_cbranch_execnz .LBB6_1
-; GFX13-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX13-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX13-NEXT: s_set_pc_i64 s[30:31]
+; GFX13-SDAG-LABEL: v_mov_b64_double:
+; GFX13-SDAG: ; %bb.0:
+; GFX13-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX13-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX13-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX13-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX13-SDAG-NEXT: s_mov_b32 s0, 0
+; GFX13-SDAG-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX13-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX13-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX13-SDAG-NEXT: v_add_f64_e32 v[2:3], 0x4063233333333333, v[4:5]
+; GFX13-SDAG-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX13-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX13-SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
+; GFX13-SDAG-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX13-SDAG-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX13-SDAG-NEXT: s_cbranch_execnz .LBB6_1
+; GFX13-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX13-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX13-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX13-GISEL-LABEL: v_mov_b64_double:
+; GFX13-GISEL: ; %bb.0:
+; GFX13-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX13-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX13-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX13-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX13-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX13-GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX13-GISEL-NEXT: ; implicit-def: $sgpr2
+; GFX13-GISEL-NEXT: .LBB6_1: ; %atomicrmw.start
+; GFX13-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX13-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX13-GISEL-NEXT: v_add_f64_e32 v[2:3], 0x4063233333333333, v[4:5]
+; GFX13-GISEL-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX13-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX13-GISEL-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
+; GFX13-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX13-GISEL-NEXT: s_and_not1_b32 s2, s2, exec_lo
+; GFX13-GISEL-NEXT: s_and_b32 s3, exec_lo, s0
+; GFX13-GISEL-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX13-GISEL-NEXT: s_or_b32 s2, s2, s3
+; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX13-GISEL-NEXT: s_and_b32 s3, exec_lo, s2
+; GFX13-GISEL-NEXT: s_or_b32 s1, s1, s3
+; GFX13-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX13-GISEL-NEXT: s_cbranch_execnz .LBB6_1
+; GFX13-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX13-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX13-GISEL-NEXT: s_set_pc_i64 s[30:31]
%result = atomicrmw fadd ptr addrspace(1) %ptr, double 153.1 monotonic
ret void
}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w32.ll
index bf043cfc73a0b..6ae612f825a07 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w32.ll
@@ -348,25 +348,31 @@ define amdgpu_cs_chain void @control_flow(<3 x i32> inreg %sgpr, ptr inreg %call
; GISEL12-NEXT: ; %bb.1: ; %shader.preheader
; GISEL12-NEXT: v_add_nc_u32_e32 v1, -1, v12
; GISEL12-NEXT: s_mov_b32 s4, 0
+; GISEL12-NEXT: ; implicit-def: $sgpr8
; GISEL12-NEXT: .LBB3_2: ; %shader
; GISEL12-NEXT: ; =>This Inner Loop Header: Depth=1
; GISEL12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GISEL12-NEXT: v_mov_b32_e32 v2, v1
; GISEL12-NEXT: v_add_nc_u32_e32 v1, 1, v2
-; GISEL12-NEXT: s_or_saveexec_b32 s8, -1
+; GISEL12-NEXT: s_or_saveexec_b32 s9, -1
; GISEL12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GISEL12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL12-NEXT: v_cndmask_b32_e64 v0, 0x47, v1, s8
-; GISEL12-NEXT: v_cmp_ne_u32_e64 s9, 0, v0
-; GISEL12-NEXT: s_mov_b32 exec_lo, s8
+; GISEL12-NEXT: v_cndmask_b32_e64 v0, 0x47, v1, s9
+; GISEL12-NEXT: v_cmp_ne_u32_e64 s10, 0, v0
+; GISEL12-NEXT: s_mov_b32 exec_lo, s9
; GISEL12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v13, v1
-; GISEL12-NEXT: v_mov_b32_e32 v11, s9
+; GISEL12-NEXT: v_mov_b32_e32 v11, s10
; GISEL12-NEXT: s_or_b32 s4, vcc_lo, s4
+; GISEL12-NEXT: s_and_not1_b32 s8, s8, exec_lo
; GISEL12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL12-NEXT: s_and_b32 s9, exec_lo, s4
+; GISEL12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL12-NEXT: s_or_b32 s8, s8, s9
; GISEL12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GISEL12-NEXT: s_cbranch_execnz .LBB3_2
; GISEL12-NEXT: ; %bb.3: ; %tail.loopexit
-; GISEL12-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GISEL12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL12-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GISEL12-NEXT: v_add_nc_u32_e32 v10, 43, v2
; GISEL12-NEXT: .LBB3_4: ; %Flow1
; GISEL12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -463,21 +469,26 @@ define amdgpu_cs_chain void @control_flow(<3 x i32> inreg %sgpr, ptr inreg %call
; GISEL10-NEXT: ; %bb.1: ; %shader.preheader
; GISEL10-NEXT: v_add_nc_u32_e32 v1, -1, v12
; GISEL10-NEXT: s_mov_b32 s4, 0
+; GISEL10-NEXT: ; implicit-def: $sgpr8
+; GISEL10-NEXT: .p2align 6
; GISEL10-NEXT: .LBB3_2: ; %shader
; GISEL10-NEXT: ; =>This Inner Loop Header: Depth=1
; GISEL10-NEXT: v_mov_b32_e32 v2, v1
; GISEL10-NEXT: v_add_nc_u32_e32 v1, 1, v2
-; GISEL10-NEXT: s_or_saveexec_b32 s8, -1
-; GISEL10-NEXT: v_cndmask_b32_e64 v0, 0x47, v1, s8
-; GISEL10-NEXT: v_cmp_ne_u32_e64 s9, 0, v0
-; GISEL10-NEXT: s_mov_b32 exec_lo, s8
+; GISEL10-NEXT: s_or_saveexec_b32 s9, -1
+; GISEL10-NEXT: v_cndmask_b32_e64 v0, 0x47, v1, s9
+; GISEL10-NEXT: v_cmp_ne_u32_e64 s10, 0, v0
+; GISEL10-NEXT: s_mov_b32 exec_lo, s9
; GISEL10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v13, v1
-; GISEL10-NEXT: v_mov_b32_e32 v11, s9
+; GISEL10-NEXT: v_mov_b32_e32 v11, s10
; GISEL10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GISEL10-NEXT: s_andn2_b32 s8, s8, exec_lo
+; GISEL10-NEXT: s_and_b32 s9, exec_lo, s4
+; GISEL10-NEXT: s_or_b32 s8, s8, s9
; GISEL10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GISEL10-NEXT: s_cbranch_execnz .LBB3_2
; GISEL10-NEXT: ; %bb.3: ; %tail.loopexit
-; GISEL10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GISEL10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GISEL10-NEXT: v_add_nc_u32_e32 v10, 43, v2
; GISEL10-NEXT: .LBB3_4: ; %Flow1
; GISEL10-NEXT: s_or_b32 exec_lo, exec_lo, s3
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.atomic.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.atomic.buffer.load.ll
index 6dc91c74abace..2394402523e4a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.atomic.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.atomic.buffer.load.ll
@@ -11,45 +11,203 @@
; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -global-isel=1 | FileCheck %s -check-prefixes=GFX12,GFX12-GISEL-FAKE16
define amdgpu_kernel void @raw_atomic_buffer_load_i32(<4 x i32> %addr) {
-; GFX11-LABEL: raw_atomic_buffer_load_i32:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB0_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB0_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: raw_atomic_buffer_load_i32:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB0_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB0_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
+; GFX11-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_i32:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: .LBB0_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_i32:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: .LBB0_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_i32:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: .LBB0_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_i32:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: .LBB0_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: raw_atomic_buffer_load_i32:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: .LBB0_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_i32:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: .LBB0_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_i32:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: .LBB0_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_i32:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: .LBB0_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_i32:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: .LBB0_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
; GFX12-TRUE16-LABEL: raw_atomic_buffer_load_i32:
; GFX12-TRUE16: ; %bb.0: ; %bb
; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
@@ -100,683 +258,2066 @@ bb2:
}
define amdgpu_kernel void @raw_atomic_buffer_load_i32_off(<4 x i32> %addr) {
-; GFX11-LABEL: raw_atomic_buffer_load_i32_off:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB1_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB1_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: raw_atomic_buffer_load_i32_off:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB1_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB1_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
-; GFX12-TRUE16-LABEL: raw_atomic_buffer_load_i32_off:
-; GFX12-TRUE16: ; %bb.0: ; %bb
-; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-TRUE16-NEXT: .LBB1_1: ; %bb1
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %bb2
-; GFX12-TRUE16-NEXT: s_endpgm
-; GFX12-FAKE16-LABEL: raw_atomic_buffer_load_i32_off:
-; GFX12-FAKE16: ; %bb.0: ; %bb
-; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-FAKE16-NEXT: .LBB1_1: ; %bb1
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB1_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %bb2
-; GFX12-FAKE16-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- br label %bb1
-bb1:
- %load = call i32 @llvm.amdgcn.raw.atomic.buffer.load.i32(<4 x i32> %addr, i32 0, i32 0, i32 1)
- %cmp = icmp eq i32 %load, %id
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-define amdgpu_kernel void @raw_atomic_buffer_load_i32_soff(<4 x i32> %addr) {
-; GFX11-LABEL: raw_atomic_buffer_load_i32_soff:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB2_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b32 v1, off, s[0:3], 4 offset:4 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB2_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: raw_atomic_buffer_load_i32_soff:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: s_mov_b32 s5, 4
-; GFX12-NEXT: .LBB2_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v1, off, s[0:3], s5 offset:4 th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB2_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
-; GFX12-TRUE16-LABEL: raw_atomic_buffer_load_i32_soff:
-; GFX12-TRUE16: ; %bb.0: ; %bb
-; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-TRUE16-NEXT: s_mov_b32 s5, 4
-; GFX12-TRUE16-NEXT: .LBB2_1: ; %bb1
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], s5 offset:4 th:TH_LOAD_NT
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB2_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %bb2
-; GFX12-TRUE16-NEXT: s_endpgm
-; GFX12-FAKE16-LABEL: raw_atomic_buffer_load_i32_soff:
-; GFX12-FAKE16: ; %bb.0: ; %bb
-; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-FAKE16-NEXT: s_mov_b32 s5, 4
-; GFX12-FAKE16-NEXT: .LBB2_1: ; %bb1
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], s5 offset:4 th:TH_LOAD_NT
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB2_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %bb2
-; GFX12-FAKE16-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- br label %bb1
-bb1:
- %load = call i32 @llvm.amdgcn.raw.atomic.buffer.load.i32(<4 x i32> %addr, i32 4, i32 4, i32 1)
- %cmp = icmp eq i32 %load, %id
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-define amdgpu_kernel void @raw_atomic_buffer_load_i32_dlc(<4 x i32> %addr) {
-; GFX11-LABEL: raw_atomic_buffer_load_i32_dlc:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB3_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 dlc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB3_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: raw_atomic_buffer_load_i32_dlc:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB3_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT_RT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB3_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
-; GFX12-TRUE16-LABEL: raw_atomic_buffer_load_i32_dlc:
-; GFX12-TRUE16: ; %bb.0: ; %bb
-; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-TRUE16-NEXT: .LBB3_1: ; %bb1
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT_RT
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB3_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %bb2
-; GFX12-TRUE16-NEXT: s_endpgm
-; GFX12-FAKE16-LABEL: raw_atomic_buffer_load_i32_dlc:
-; GFX12-FAKE16: ; %bb.0: ; %bb
-; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX11-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_i32_off:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: .LBB1_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_i32_off:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: .LBB1_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_i32_off:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: .LBB1_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_i32_off:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: .LBB1_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: raw_atomic_buffer_load_i32_off:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: .LBB1_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB1_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_i32_off:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: .LBB1_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_i32_off:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: .LBB1_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_i32_off:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: .LBB1_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_i32_off:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: .LBB1_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: raw_atomic_buffer_load_i32_off:
+; GFX12-TRUE16: ; %bb.0: ; %bb
+; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-TRUE16-NEXT: .LBB1_1: ; %bb1
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-TRUE16-NEXT: s_endpgm
+; GFX12-FAKE16-LABEL: raw_atomic_buffer_load_i32_off:
+; GFX12-FAKE16: ; %bb.0: ; %bb
+; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-FAKE16-NEXT: .LBB1_1: ; %bb1
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ br label %bb1
+bb1:
+ %load = call i32 @llvm.amdgcn.raw.atomic.buffer.load.i32(<4 x i32> %addr, i32 0, i32 0, i32 1)
+ %cmp = icmp eq i32 %load, %id
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+define amdgpu_kernel void @raw_atomic_buffer_load_i32_soff(<4 x i32> %addr) {
+; GFX11-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_i32_soff:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: .LBB2_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 4 offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_i32_soff:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: .LBB2_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 4 offset:4 glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_i32_soff:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: .LBB2_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 4 offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_i32_soff:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: .LBB2_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 4 offset:4 glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: raw_atomic_buffer_load_i32_soff:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: .LBB2_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_load_b32 v1, off, s[0:3], 4 offset:4 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB2_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_i32_soff:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s5, 4
+; GFX12-SDAG-TRUE16-NEXT: .LBB2_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], s5 offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_i32_soff:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s5, 4
+; GFX12-SDAG-FAKE16-NEXT: .LBB2_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], s5 offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_i32_soff:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s5, 4
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX12-GISEL-TRUE16-NEXT: .LBB2_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], s5 offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s6, s6, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s7, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s6, s6, s7
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_i32_soff:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s5, 4
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX12-GISEL-FAKE16-NEXT: .LBB2_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], s5 offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s6, s6, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s7, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s6, s6, s7
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: raw_atomic_buffer_load_i32_soff:
+; GFX12-TRUE16: ; %bb.0: ; %bb
+; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 4
+; GFX12-TRUE16-NEXT: .LBB2_1: ; %bb1
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], s5 offset:4 th:TH_LOAD_NT
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-TRUE16-NEXT: s_endpgm
+; GFX12-FAKE16-LABEL: raw_atomic_buffer_load_i32_soff:
+; GFX12-FAKE16: ; %bb.0: ; %bb
+; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 4
+; GFX12-FAKE16-NEXT: .LBB2_1: ; %bb1
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], s5 offset:4 th:TH_LOAD_NT
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ br label %bb1
+bb1:
+ %load = call i32 @llvm.amdgcn.raw.atomic.buffer.load.i32(<4 x i32> %addr, i32 4, i32 4, i32 1)
+ %cmp = icmp eq i32 %load, %id
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+define amdgpu_kernel void @raw_atomic_buffer_load_i32_dlc(<4 x i32> %addr) {
+; GFX11-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_i32_dlc:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: .LBB3_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 dlc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_i32_dlc:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: .LBB3_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 dlc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_i32_dlc:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: .LBB3_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 dlc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_i32_dlc:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: .LBB3_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 dlc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: raw_atomic_buffer_load_i32_dlc:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: .LBB3_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 dlc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB3_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_i32_dlc:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: .LBB3_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT_RT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_i32_dlc:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: .LBB3_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT_RT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_i32_dlc:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: .LBB3_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT_RT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_i32_dlc:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: .LBB3_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT_RT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: raw_atomic_buffer_load_i32_dlc:
+; GFX12-TRUE16: ; %bb.0: ; %bb
+; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-TRUE16-NEXT: .LBB3_1: ; %bb1
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT_RT
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-TRUE16-NEXT: s_endpgm
+; GFX12-FAKE16-LABEL: raw_atomic_buffer_load_i32_dlc:
+; GFX12-FAKE16: ; %bb.0: ; %bb
+; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-FAKE16-NEXT: .LBB3_1: ; %bb1
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT_RT
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ br label %bb1
+bb1:
+ %load = call i32 @llvm.amdgcn.raw.atomic.buffer.load.i32(<4 x i32> %addr, i32 4, i32 0, i32 4)
+ %cmp = icmp eq i32 %load, %id
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+
+define amdgpu_kernel void @raw_nonatomic_buffer_load_i32(<4 x i32> %addr) {
+; GFX11-SDAG-TRUE16-LABEL: raw_nonatomic_buffer_load_i32:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: .LBB4_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s0, s1, s0
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: raw_nonatomic_buffer_load_i32:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 glc
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-FAKE16-NEXT: .LBB4_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s0, s1, s0
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: raw_nonatomic_buffer_load_i32:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-TRUE16-NEXT: .LBB4_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s0, s2, s0
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s1, s1, s2
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: raw_nonatomic_buffer_load_i32:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-FAKE16-NEXT: .LBB4_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s0, s2, s0
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s1, s1, s2
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: raw_nonatomic_buffer_load_i32:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-NEXT: .LBB4_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s0, s2, s0
+; GFX11-GISEL-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX11-GISEL-NEXT: s_or_b32 s1, s1, s2
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB4_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: raw_nonatomic_buffer_load_i32:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-TRUE16-NEXT: .LBB4_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s0, s1, s0
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: raw_nonatomic_buffer_load_i32:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-FAKE16-NEXT: .LBB4_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s0, s1, s0
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: raw_nonatomic_buffer_load_i32:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-TRUE16-NEXT: .LBB4_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s0, s2, s0
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s1, s1, s2
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: raw_nonatomic_buffer_load_i32:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr1
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-FAKE16-NEXT: .LBB4_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s0, s2, s0
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s1, s1, s2
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: raw_nonatomic_buffer_load_i32:
+; GFX12-TRUE16: ; %bb.0: ; %bb
+; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-TRUE16-NEXT: .LBB4_1: ; %bb1
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-TRUE16-NEXT: s_endpgm
+; GFX12-FAKE16-LABEL: raw_nonatomic_buffer_load_i32:
+; GFX12-FAKE16: ; %bb.0: ; %bb
+; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-FAKE16-NEXT: .LBB4_1: ; %bb1
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, s1, s0
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ br label %bb1
+bb1:
+ %load = call i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32> %addr, i32 4, i32 0, i32 1)
+ %cmp = icmp eq i32 %load, %id
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+
+define amdgpu_kernel void @raw_atomic_buffer_load_i64(<4 x i32> %addr) {
+; GFX11-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_i64:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: .LBB5_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], 0 offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_i64:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: .LBB5_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], 0 offset:4 glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_i64:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: .LBB5_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_i64:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: .LBB5_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: raw_atomic_buffer_load_i64:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: .LBB5_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_load_b64 v[2:3], off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB5_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_i64:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_i64:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_i64:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_i64:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %id.zext = zext i32 %id to i64
+ br label %bb1
+bb1:
+ %load = call i64 @llvm.amdgcn.raw.atomic.buffer.load.i64(<4 x i32> %addr, i32 4, i32 0, i32 1)
+ %cmp = icmp eq i64 %load, %id.zext
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+
+define amdgpu_kernel void @raw_atomic_buffer_load_v2i16(<4 x i32> %addr) {
+; GFX11-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_v2i16:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_v2i16:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_v2i16:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_v2i16:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: raw_atomic_buffer_load_v2i16:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: .LBB6_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_v2i16:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_v2i16:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_v2i16:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_v2i16:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: raw_atomic_buffer_load_v2i16:
+; GFX12-TRUE16: ; %bb.0: ; %bb
+; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-TRUE16-NEXT: s_endpgm
+; GFX12-FAKE16-LABEL: raw_atomic_buffer_load_v2i16:
+; GFX12-FAKE16: ; %bb.0: ; %bb
+; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-FAKE16-NEXT: .LBB3_1: ; %bb1
+; GFX12-FAKE16-NEXT: .LBB6_1: ; %bb1
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT_RT
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
; GFX12-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX12-FAKE16-NEXT: s_endpgm
bb:
%id = tail call i32 @llvm.amdgcn.workitem.id.x()
br label %bb1
bb1:
- %load = call i32 @llvm.amdgcn.raw.atomic.buffer.load.i32(<4 x i32> %addr, i32 4, i32 0, i32 4)
- %cmp = icmp eq i32 %load, %id
+ %load = call <2 x i16> @llvm.amdgcn.raw.atomic.buffer.load.v2i16(<4 x i32> %addr, i32 0, i32 0, i32 1)
+ %bitcast = bitcast <2 x i16> %load to i32
+ %cmp = icmp eq i32 %bitcast, %id
br i1 %cmp, label %bb1, label %bb2
bb2:
ret void
}
-define amdgpu_kernel void @raw_nonatomic_buffer_load_i32(<4 x i32> %addr) {
-; GFX11-LABEL: raw_nonatomic_buffer_load_i32:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 glc
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: .LBB4_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_and_b32 s1, exec_lo, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s1, s0
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB4_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: raw_nonatomic_buffer_load_i32:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-NEXT: .LBB4_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_and_b32 s1, exec_lo, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_or_b32 s0, s1, s0
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB4_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
-; GFX12-TRUE16-LABEL: raw_nonatomic_buffer_load_i32:
-; GFX12-TRUE16: ; %bb.0: ; %bb
-; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT
-; GFX12-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-TRUE16-NEXT: .LBB4_1: ; %bb1
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB4_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %bb2
-; GFX12-TRUE16-NEXT: s_endpgm
-; GFX12-FAKE16-LABEL: raw_nonatomic_buffer_load_i32:
-; GFX12-FAKE16: ; %bb.0: ; %bb
-; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT
-; GFX12-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-FAKE16-NEXT: .LBB4_1: ; %bb1
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, s1, s0
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB4_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %bb2
-; GFX12-FAKE16-NEXT: s_endpgm
+define amdgpu_kernel void @raw_atomic_buffer_load_v4i16(<4 x i32> %addr) {
+; GFX11-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s6, v1
+; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s7, v2
+; GFX11-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s6, v1
+; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s7, v2
+; GFX11-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: .LBB7_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_readfirstlane_b32 s6, v1
+; GFX11-GISEL-NEXT: v_readfirstlane_b32 s7, v2
+; GFX11-GISEL-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
+; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
+; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
bb:
%id = tail call i32 @llvm.amdgcn.workitem.id.x()
br label %bb1
bb1:
- %load = call i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32> %addr, i32 4, i32 0, i32 1)
- %cmp = icmp eq i32 %load, %id
+ %load = call <4 x i16> @llvm.amdgcn.raw.atomic.buffer.load.v4i16(<4 x i32> %addr, i32 4, i32 0, i32 1)
+ %shortened = shufflevector <4 x i16> %load, <4 x i16> poison, <2 x i32> <i32 0, i32 2>
+ %bitcast = bitcast <2 x i16> %shortened to i32
+ %cmp = icmp eq i32 %bitcast, %id
br i1 %cmp, label %bb1, label %bb2
bb2:
ret void
}
-define amdgpu_kernel void @raw_atomic_buffer_load_i64(<4 x i32> %addr) {
-; GFX11-LABEL: raw_atomic_buffer_load_i64:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB5_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b64 v[2:3], off, s[0:3], 0 offset:4 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB5_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
+define amdgpu_kernel void @raw_atomic_buffer_load_v4i32(<4 x i32> %addr) {
+; GFX11-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_v4i32:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b128 v[1:4], off, s[0:3], 0 offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v4, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
;
-; GFX12-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_i64:
+; GFX11-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_v4i32:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b128 v[1:4], off, s[0:3], 0 offset:4 glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v4, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_v4i32:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b128 v[1:4], off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v4, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_v4i32:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b128 v[1:4], off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v4, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: raw_atomic_buffer_load_v4i32:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: .LBB8_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_load_b128 v[1:4], off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v4, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_v4i32:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
; GFX12-SDAG-TRUE16-NEXT: global_wb
; GFX12-SDAG-TRUE16-NEXT: v_nop
; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-SDAG-TRUE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: .LBB8_1: ; %bb1
; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b128 v[2:5], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX12-SDAG-TRUE16-NEXT: s_endpgm
;
-; GFX12-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_i64:
+; GFX12-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_v4i32:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
; GFX12-SDAG-FAKE16-NEXT: global_wb
; GFX12-SDAG-FAKE16-NEXT: v_nop
; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-SDAG-FAKE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: .LBB8_1: ; %bb1
; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b128 v[2:5], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX12-SDAG-FAKE16-NEXT: s_endpgm
;
-; GFX12-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_i64:
+; GFX12-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_v4i32:
; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
; GFX12-GISEL-TRUE16-NEXT: global_wb
; GFX12-GISEL-TRUE16-NEXT: v_nop
; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-GISEL-TRUE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: .LBB8_1: ; %bb1
; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b128 v[2:5], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX12-GISEL-TRUE16-NEXT: s_endpgm
;
-; GFX12-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_i64:
+; GFX12-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_v4i32:
; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
; GFX12-GISEL-FAKE16-NEXT: global_wb
; GFX12-GISEL-FAKE16-NEXT: v_nop
; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-GISEL-FAKE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: .LBB8_1: ; %bb1
; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b128 v[2:5], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX12-GISEL-FAKE16-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- %id.zext = zext i32 %id to i64
- br label %bb1
-bb1:
- %load = call i64 @llvm.amdgcn.raw.atomic.buffer.load.i64(<4 x i32> %addr, i32 4, i32 0, i32 1)
- %cmp = icmp eq i64 %load, %id.zext
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-
-define amdgpu_kernel void @raw_atomic_buffer_load_v2i16(<4 x i32> %addr) {
-; GFX11-LABEL: raw_atomic_buffer_load_v2i16:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB6_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB6_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: raw_atomic_buffer_load_v2i16:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB6_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB6_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
-; GFX12-TRUE16-LABEL: raw_atomic_buffer_load_v2i16:
+; GFX12-TRUE16-LABEL: raw_atomic_buffer_load_v4i32:
; GFX12-TRUE16: ; %bb.0: ; %bb
; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-TRUE16-NEXT: .LBB8_1: ; %bb1
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-TRUE16-NEXT: buffer_load_b128 v[2:5], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX12-TRUE16-NEXT: s_endpgm
-; GFX12-FAKE16-LABEL: raw_atomic_buffer_load_v2i16:
+; GFX12-FAKE16-LABEL: raw_atomic_buffer_load_v4i32:
; GFX12-FAKE16: ; %bb.0: ; %bb
; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-FAKE16-NEXT: .LBB8_1: ; %bb1
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-FAKE16-NEXT: buffer_load_b128 v[2:5], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX12-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX12-FAKE16-NEXT: s_endpgm
bb:
%id = tail call i32 @llvm.amdgcn.workitem.id.x()
br label %bb1
bb1:
- %load = call <2 x i16> @llvm.amdgcn.raw.atomic.buffer.load.v2i16(<4 x i32> %addr, i32 0, i32 0, i32 1)
- %bitcast = bitcast <2 x i16> %load to i32
- %cmp = icmp eq i32 %bitcast, %id
+ %load = call <4 x i32> @llvm.amdgcn.raw.atomic.buffer.load.v4i32(<4 x i32> %addr, i32 4, i32 0, i32 1)
+ %extracted = extractelement <4 x i32> %load, i32 3
+ %cmp = icmp eq i32 %extracted, %id
br i1 %cmp, label %bb1, label %bb2
bb2:
ret void
}
-define amdgpu_kernel void @raw_atomic_buffer_load_v4i16(<4 x i32> %addr) {
-; GFX11-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_v4i16:
+define amdgpu_kernel void @raw_atomic_buffer_load_ptr(<4 x i32> %addr) {
+; GFX11-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_ptr:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-SDAG-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: .LBB9_1: ; %bb1
; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: flat_load_b32 v1, v[1:2]
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX11-SDAG-TRUE16-NEXT: s_endpgm
;
-; GFX11-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX11-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_ptr:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-SDAG-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: .LBB9_1: ; %bb1
; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX11-SDAG-FAKE16-NEXT: flat_load_b32 v1, v[1:2]
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX11-SDAG-FAKE16-NEXT: s_endpgm
;
-; GFX11-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX11-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_ptr:
; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-GISEL-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: .LBB9_1: ; %bb1
; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-TRUE16-NEXT: buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX11-GISEL-TRUE16-NEXT: flat_load_b32 v1, v[1:2]
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX11-GISEL-TRUE16-NEXT: s_endpgm
;
-; GFX11-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX11-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_ptr:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-GISEL-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: .LBB9_1: ; %bb1
; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-FAKE16-NEXT: buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX11-GISEL-FAKE16-NEXT: flat_load_b32 v1, v[1:2]
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX11-GISEL-FAKE16-NEXT: s_endpgm
;
-; GFX11-GISEL-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX11-GISEL-LABEL: raw_atomic_buffer_load_ptr:
; GFX11-GISEL: ; %bb.0: ; %bb
; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
-; GFX11-GISEL-NEXT: .LBB7_1: ; %bb1
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: .LBB9_1: ; %bb1
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-GISEL-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-GISEL-NEXT: s_pack_ll_b32_b16 s5, s5, s6
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX11-GISEL-NEXT: flat_load_b32 v1, v[1:2]
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
; GFX11-GISEL-NEXT: s_endpgm
;
-; GFX12-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX12-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_ptr:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
; GFX12-SDAG-TRUE16-NEXT: global_wb
; GFX12-SDAG-TRUE16-NEXT: v_nop
@@ -785,21 +2326,22 @@ define amdgpu_kernel void @raw_atomic_buffer_load_v4i16(<4 x i32> %addr) {
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-SDAG-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: .LBB9_1: ; %bb1
; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-TRUE16-NEXT: flat_load_b32 v1, v[2:3]
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX12-SDAG-TRUE16-NEXT: s_endpgm
;
-; GFX12-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX12-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_ptr:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
; GFX12-SDAG-FAKE16-NEXT: global_wb
; GFX12-SDAG-FAKE16-NEXT: v_nop
@@ -808,23 +2350,22 @@ define amdgpu_kernel void @raw_atomic_buffer_load_v4i16(<4 x i32> %addr) {
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-SDAG-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: .LBB9_1: ; %bb1
; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX12-SDAG-FAKE16-NEXT: flat_load_b32 v1, v[2:3]
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX12-SDAG-FAKE16-NEXT: s_endpgm
;
-; GFX12-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX12-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_ptr:
; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
; GFX12-GISEL-TRUE16-NEXT: global_wb
; GFX12-GISEL-TRUE16-NEXT: v_nop
@@ -833,23 +2374,26 @@ define amdgpu_kernel void @raw_atomic_buffer_load_v4i16(<4 x i32> %addr) {
; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-GISEL-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: .LBB9_1: ; %bb1
; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s5, v2
-; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s6, v3
-; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX12-GISEL-TRUE16-NEXT: flat_load_b32 v1, v[2:3]
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX12-GISEL-TRUE16-NEXT: s_endpgm
;
-; GFX12-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX12-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_ptr:
; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
; GFX12-GISEL-FAKE16-NEXT: global_wb
; GFX12-GISEL-FAKE16-NEXT: v_nop
@@ -858,168 +2402,24 @@ define amdgpu_kernel void @raw_atomic_buffer_load_v4i16(<4 x i32> %addr) {
; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-GISEL-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: .LBB9_1: ; %bb1
; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s5, v2
-; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s6, v3
-; GFX12-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX12-GISEL-FAKE16-NEXT: flat_load_b32 v1, v[2:3]
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX12-GISEL-FAKE16-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- br label %bb1
-bb1:
- %load = call <4 x i16> @llvm.amdgcn.raw.atomic.buffer.load.v4i16(<4 x i32> %addr, i32 4, i32 0, i32 1)
- %shortened = shufflevector <4 x i16> %load, <4 x i16> poison, <2 x i32> <i32 0, i32 2>
- %bitcast = bitcast <2 x i16> %shortened to i32
- %cmp = icmp eq i32 %bitcast, %id
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-
-define amdgpu_kernel void @raw_atomic_buffer_load_v4i32(<4 x i32> %addr) {
-; GFX11-LABEL: raw_atomic_buffer_load_v4i32:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB8_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b128 v[1:4], off, s[0:3], 0 offset:4 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v4, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB8_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: raw_atomic_buffer_load_v4i32:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB8_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: buffer_load_b128 v[2:5], off, s[0:3], null offset:4 th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB8_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
-; GFX12-TRUE16-LABEL: raw_atomic_buffer_load_v4i32:
-; GFX12-TRUE16: ; %bb.0: ; %bb
-; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-TRUE16-NEXT: .LBB8_1: ; %bb1
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b128 v[2:5], off, s[0:3], null offset:4 th:TH_LOAD_NT
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
-; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %bb2
-; GFX12-TRUE16-NEXT: s_endpgm
-; GFX12-FAKE16-LABEL: raw_atomic_buffer_load_v4i32:
-; GFX12-FAKE16: ; %bb.0: ; %bb
-; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-FAKE16-NEXT: .LBB8_1: ; %bb1
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b128 v[2:5], off, s[0:3], null offset:4 th:TH_LOAD_NT
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
-; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %bb2
-; GFX12-FAKE16-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- br label %bb1
-bb1:
- %load = call <4 x i32> @llvm.amdgcn.raw.atomic.buffer.load.v4i32(<4 x i32> %addr, i32 4, i32 0, i32 1)
- %extracted = extractelement <4 x i32> %load, i32 3
- %cmp = icmp eq i32 %extracted, %id
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-
-define amdgpu_kernel void @raw_atomic_buffer_load_ptr(<4 x i32> %addr) {
-; GFX11-LABEL: raw_atomic_buffer_load_ptr:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB9_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: flat_load_b32 v1, v[1:2]
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB9_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: raw_atomic_buffer_load_ptr:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB9_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: flat_load_b32 v1, v[2:3]
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB9_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
; GFX12-TRUE16-LABEL: raw_atomic_buffer_load_ptr:
; GFX12-TRUE16: ; %bb.0: ; %bb
; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
@@ -1083,3 +2483,6 @@ declare <4 x i32> @llvm.amdgcn.raw.atomic.buffer.load.v4i32(<4 x i32>, i32, i32,
declare ptr @llvm.amdgcn.raw.atomic.buffer.load.ptr(<4 x i32>, i32, i32, i32 immarg)
declare i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32>, i32, i32, i32 immarg)
declare i32 @llvm.amdgcn.workitem.id.x()
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11: {{.*}}
+; GFX12: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll
index eb06520422c77..04efe701a95d7 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll
@@ -11,45 +11,203 @@
; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -global-isel=1 | FileCheck %s -check-prefixes=GFX12,GFX12-GISEL-FAKE16
define amdgpu_kernel void @raw_ptr_atomic_buffer_ptr_load_i32(ptr addrspace(8) %ptr) {
-; GFX11-LABEL: raw_ptr_atomic_buffer_ptr_load_i32:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB0_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB0_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: raw_ptr_atomic_buffer_ptr_load_i32:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB0_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB0_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
+; GFX11-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_ptr_load_i32:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: .LBB0_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_ptr_load_i32:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: .LBB0_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_ptr_load_i32:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: .LBB0_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_ptr_load_i32:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: .LBB0_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: raw_ptr_atomic_buffer_ptr_load_i32:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: .LBB0_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_ptr_load_i32:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: .LBB0_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_ptr_load_i32:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: .LBB0_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_ptr_load_i32:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: .LBB0_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_ptr_load_i32:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: .LBB0_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
; GFX12-TRUE16-LABEL: raw_ptr_atomic_buffer_ptr_load_i32:
; GFX12-TRUE16: ; %bb.0: ; %bb
; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
@@ -100,683 +258,2066 @@ bb2:
}
define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i32_off(ptr addrspace(8) %ptr) {
-; GFX11-LABEL: raw_ptr_atomic_buffer_load_i32_off:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB1_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB1_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: raw_ptr_atomic_buffer_load_i32_off:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB1_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB1_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
-; GFX12-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i32_off:
-; GFX12-TRUE16: ; %bb.0: ; %bb
-; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-TRUE16-NEXT: .LBB1_1: ; %bb1
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %bb2
-; GFX12-TRUE16-NEXT: s_endpgm
-; GFX12-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i32_off:
-; GFX12-FAKE16: ; %bb.0: ; %bb
-; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-FAKE16-NEXT: .LBB1_1: ; %bb1
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB1_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %bb2
-; GFX12-FAKE16-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- br label %bb1
-bb1:
- %load = call i32 @llvm.amdgcn.raw.ptr.atomic.buffer.load.i32(ptr addrspace(8) %ptr, i32 0, i32 0, i32 1)
- %cmp = icmp eq i32 %load, %id
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i32_soff(ptr addrspace(8) %ptr) {
-; GFX11-LABEL: raw_ptr_atomic_buffer_load_i32_soff:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB2_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b32 v1, off, s[0:3], 4 offset:4 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB2_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: raw_ptr_atomic_buffer_load_i32_soff:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: s_mov_b32 s5, 4
-; GFX12-NEXT: .LBB2_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v1, off, s[0:3], s5 offset:4 th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB2_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
-; GFX12-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i32_soff:
-; GFX12-TRUE16: ; %bb.0: ; %bb
-; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-TRUE16-NEXT: s_mov_b32 s5, 4
-; GFX12-TRUE16-NEXT: .LBB2_1: ; %bb1
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], s5 offset:4 th:TH_LOAD_NT
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB2_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %bb2
-; GFX12-TRUE16-NEXT: s_endpgm
-; GFX12-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i32_soff:
-; GFX12-FAKE16: ; %bb.0: ; %bb
-; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-FAKE16-NEXT: s_mov_b32 s5, 4
-; GFX12-FAKE16-NEXT: .LBB2_1: ; %bb1
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], s5 offset:4 th:TH_LOAD_NT
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB2_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %bb2
-; GFX12-FAKE16-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- br label %bb1
-bb1:
- %load = call i32 @llvm.amdgcn.raw.ptr.atomic.buffer.load.i32(ptr addrspace(8) %ptr, i32 4, i32 4, i32 1)
- %cmp = icmp eq i32 %load, %id
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i32_dlc(ptr addrspace(8) %ptr) {
-; GFX11-LABEL: raw_ptr_atomic_buffer_load_i32_dlc:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB3_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 dlc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB3_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: raw_ptr_atomic_buffer_load_i32_dlc:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB3_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT_RT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB3_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
-; GFX12-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i32_dlc:
-; GFX12-TRUE16: ; %bb.0: ; %bb
-; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-TRUE16-NEXT: .LBB3_1: ; %bb1
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT_RT
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB3_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %bb2
-; GFX12-TRUE16-NEXT: s_endpgm
-; GFX12-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i32_dlc:
-; GFX12-FAKE16: ; %bb.0: ; %bb
-; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX11-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i32_off:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: .LBB1_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i32_off:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: .LBB1_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i32_off:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: .LBB1_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i32_off:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: .LBB1_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: raw_ptr_atomic_buffer_load_i32_off:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: .LBB1_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB1_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i32_off:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: .LBB1_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i32_off:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: .LBB1_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i32_off:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: .LBB1_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i32_off:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: .LBB1_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i32_off:
+; GFX12-TRUE16: ; %bb.0: ; %bb
+; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-TRUE16-NEXT: .LBB1_1: ; %bb1
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-TRUE16-NEXT: s_endpgm
+; GFX12-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i32_off:
+; GFX12-FAKE16: ; %bb.0: ; %bb
+; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-FAKE16-NEXT: .LBB1_1: ; %bb1
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ br label %bb1
+bb1:
+ %load = call i32 @llvm.amdgcn.raw.ptr.atomic.buffer.load.i32(ptr addrspace(8) %ptr, i32 0, i32 0, i32 1)
+ %cmp = icmp eq i32 %load, %id
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i32_soff(ptr addrspace(8) %ptr) {
+; GFX11-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i32_soff:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: .LBB2_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 4 offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i32_soff:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: .LBB2_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 4 offset:4 glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i32_soff:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: .LBB2_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 4 offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i32_soff:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: .LBB2_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 4 offset:4 glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: raw_ptr_atomic_buffer_load_i32_soff:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: .LBB2_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_load_b32 v1, off, s[0:3], 4 offset:4 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB2_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i32_soff:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s5, 4
+; GFX12-SDAG-TRUE16-NEXT: .LBB2_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], s5 offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i32_soff:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s5, 4
+; GFX12-SDAG-FAKE16-NEXT: .LBB2_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], s5 offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i32_soff:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s5, 4
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX12-GISEL-TRUE16-NEXT: .LBB2_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], s5 offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s6, s6, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s7, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s6, s6, s7
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i32_soff:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s5, 4
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX12-GISEL-FAKE16-NEXT: .LBB2_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], s5 offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s6, s6, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s7, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s6, s6, s7
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i32_soff:
+; GFX12-TRUE16: ; %bb.0: ; %bb
+; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 4
+; GFX12-TRUE16-NEXT: .LBB2_1: ; %bb1
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], s5 offset:4 th:TH_LOAD_NT
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-TRUE16-NEXT: s_endpgm
+; GFX12-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i32_soff:
+; GFX12-FAKE16: ; %bb.0: ; %bb
+; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 4
+; GFX12-FAKE16-NEXT: .LBB2_1: ; %bb1
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], s5 offset:4 th:TH_LOAD_NT
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ br label %bb1
+bb1:
+ %load = call i32 @llvm.amdgcn.raw.ptr.atomic.buffer.load.i32(ptr addrspace(8) %ptr, i32 4, i32 4, i32 1)
+ %cmp = icmp eq i32 %load, %id
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i32_dlc(ptr addrspace(8) %ptr) {
+; GFX11-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i32_dlc:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: .LBB3_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 dlc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i32_dlc:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: .LBB3_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 dlc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i32_dlc:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: .LBB3_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 dlc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i32_dlc:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: .LBB3_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 dlc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: raw_ptr_atomic_buffer_load_i32_dlc:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: .LBB3_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 dlc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB3_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i32_dlc:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: .LBB3_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT_RT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i32_dlc:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: .LBB3_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT_RT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i32_dlc:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: .LBB3_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT_RT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i32_dlc:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: .LBB3_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT_RT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i32_dlc:
+; GFX12-TRUE16: ; %bb.0: ; %bb
+; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-TRUE16-NEXT: .LBB3_1: ; %bb1
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT_RT
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-TRUE16-NEXT: s_endpgm
+; GFX12-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i32_dlc:
+; GFX12-FAKE16: ; %bb.0: ; %bb
+; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-FAKE16-NEXT: .LBB3_1: ; %bb1
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT_RT
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ br label %bb1
+bb1:
+ %load = call i32 @llvm.amdgcn.raw.ptr.atomic.buffer.load.i32(ptr addrspace(8) %ptr, i32 4, i32 0, i32 4)
+ %cmp = icmp eq i32 %load, %id
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+
+define amdgpu_kernel void @raw_nonptr_atomic_buffer_load_i32(ptr addrspace(8) %ptr) {
+; GFX11-SDAG-TRUE16-LABEL: raw_nonptr_atomic_buffer_load_i32:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: .LBB4_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s0, s1, s0
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: raw_nonptr_atomic_buffer_load_i32:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 glc
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-FAKE16-NEXT: .LBB4_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s0, s1, s0
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: raw_nonptr_atomic_buffer_load_i32:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-TRUE16-NEXT: .LBB4_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s0, s2, s0
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s1, s1, s2
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: raw_nonptr_atomic_buffer_load_i32:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-FAKE16-NEXT: .LBB4_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s0, s2, s0
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s1, s1, s2
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: raw_nonptr_atomic_buffer_load_i32:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-NEXT: .LBB4_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s0, s2, s0
+; GFX11-GISEL-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX11-GISEL-NEXT: s_or_b32 s1, s1, s2
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB4_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: raw_nonptr_atomic_buffer_load_i32:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-TRUE16-NEXT: .LBB4_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s0, s1, s0
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: raw_nonptr_atomic_buffer_load_i32:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-FAKE16-NEXT: .LBB4_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s0, s1, s0
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: raw_nonptr_atomic_buffer_load_i32:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-TRUE16-NEXT: .LBB4_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s0, s2, s0
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s1, s1, s2
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: raw_nonptr_atomic_buffer_load_i32:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr1
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-FAKE16-NEXT: .LBB4_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s0, s2, s0
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s1, s1, s2
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: raw_nonptr_atomic_buffer_load_i32:
+; GFX12-TRUE16: ; %bb.0: ; %bb
+; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-TRUE16-NEXT: .LBB4_1: ; %bb1
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-TRUE16-NEXT: s_endpgm
+; GFX12-FAKE16-LABEL: raw_nonptr_atomic_buffer_load_i32:
+; GFX12-FAKE16: ; %bb.0: ; %bb
+; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-FAKE16-NEXT: .LBB4_1: ; %bb1
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, s1, s0
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ br label %bb1
+bb1:
+ %load = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) %ptr, i32 4, i32 0, i32 1)
+ %cmp = icmp eq i32 %load, %id
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+
+define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i64(ptr addrspace(8) %ptr) {
+; GFX11-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i64:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: .LBB5_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], 0 offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i64:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: .LBB5_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], 0 offset:4 glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i64:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: .LBB5_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i64:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: .LBB5_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: raw_ptr_atomic_buffer_load_i64:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: .LBB5_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_load_b64 v[2:3], off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB5_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i64:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i64:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i64:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i64:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %id.zext = zext i32 %id to i64
+ br label %bb1
+bb1:
+ %load = call i64 @llvm.amdgcn.raw.ptr.atomic.buffer.load.i64(ptr addrspace(8) %ptr, i32 4, i32 0, i32 1)
+ %cmp = icmp eq i64 %load, %id.zext
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+
+define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v2i16(ptr addrspace(8) %ptr) {
+; GFX11-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v2i16:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v2i16:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v2i16:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v2i16:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: raw_ptr_atomic_buffer_load_v2i16:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: .LBB6_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v2i16:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v2i16:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v2i16:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v2i16:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+; GFX12-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v2i16:
+; GFX12-TRUE16: ; %bb.0: ; %bb
+; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-TRUE16-NEXT: s_endpgm
+; GFX12-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v2i16:
+; GFX12-FAKE16: ; %bb.0: ; %bb
+; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-FAKE16-NEXT: .LBB3_1: ; %bb1
+; GFX12-FAKE16-NEXT: .LBB6_1: ; %bb1
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT_RT
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
; GFX12-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX12-FAKE16-NEXT: s_endpgm
bb:
%id = tail call i32 @llvm.amdgcn.workitem.id.x()
br label %bb1
bb1:
- %load = call i32 @llvm.amdgcn.raw.ptr.atomic.buffer.load.i32(ptr addrspace(8) %ptr, i32 4, i32 0, i32 4)
- %cmp = icmp eq i32 %load, %id
+ %load = call <2 x i16> @llvm.amdgcn.raw.ptr.atomic.buffer.load.v2i16(ptr addrspace(8) %ptr, i32 0, i32 0, i32 1)
+ %bitcast = bitcast <2 x i16> %load to i32
+ %cmp = icmp eq i32 %bitcast, %id
br i1 %cmp, label %bb1, label %bb2
bb2:
ret void
}
-define amdgpu_kernel void @raw_nonptr_atomic_buffer_load_i32(ptr addrspace(8) %ptr) {
-; GFX11-LABEL: raw_nonptr_atomic_buffer_load_i32:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:4 glc
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: .LBB4_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_and_b32 s1, exec_lo, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s1, s0
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB4_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: raw_nonptr_atomic_buffer_load_i32:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-NEXT: .LBB4_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_and_b32 s1, exec_lo, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_or_b32 s0, s1, s0
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB4_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
-; GFX12-TRUE16-LABEL: raw_nonptr_atomic_buffer_load_i32:
-; GFX12-TRUE16: ; %bb.0: ; %bb
-; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT
-; GFX12-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-TRUE16-NEXT: .LBB4_1: ; %bb1
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB4_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %bb2
-; GFX12-TRUE16-NEXT: s_endpgm
-; GFX12-FAKE16-LABEL: raw_nonptr_atomic_buffer_load_i32:
-; GFX12-FAKE16: ; %bb.0: ; %bb
-; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT
-; GFX12-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-FAKE16-NEXT: .LBB4_1: ; %bb1
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, s1, s0
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB4_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %bb2
-; GFX12-FAKE16-NEXT: s_endpgm
+define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v4i16(ptr addrspace(8) %ptr) {
+; GFX11-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s6, v1
+; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s7, v2
+; GFX11-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s6, v1
+; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s7, v2
+; GFX11-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: .LBB7_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_readfirstlane_b32 s6, v1
+; GFX11-GISEL-NEXT: v_readfirstlane_b32 s7, v2
+; GFX11-GISEL-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
+; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
+; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
bb:
%id = tail call i32 @llvm.amdgcn.workitem.id.x()
br label %bb1
bb1:
- %load = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) %ptr, i32 4, i32 0, i32 1)
- %cmp = icmp eq i32 %load, %id
+ %load = call <4 x i16> @llvm.amdgcn.raw.ptr.atomic.buffer.load.v4i16(ptr addrspace(8) %ptr, i32 4, i32 0, i32 1)
+ %shortened = shufflevector <4 x i16> %load, <4 x i16> poison, <2 x i32> <i32 0, i32 2>
+ %bitcast = bitcast <2 x i16> %shortened to i32
+ %cmp = icmp eq i32 %bitcast, %id
br i1 %cmp, label %bb1, label %bb2
bb2:
ret void
}
-define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i64(ptr addrspace(8) %ptr) {
-; GFX11-LABEL: raw_ptr_atomic_buffer_load_i64:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB5_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b64 v[2:3], off, s[0:3], 0 offset:4 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB5_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
+define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v4i32(ptr addrspace(8) %ptr) {
+; GFX11-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v4i32:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b128 v[1:4], off, s[0:3], 0 offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v4, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
;
-; GFX12-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i64:
+; GFX11-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v4i32:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b128 v[1:4], off, s[0:3], 0 offset:4 glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v4, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v4i32:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b128 v[1:4], off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v4, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v4i32:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b128 v[1:4], off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v4, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: raw_ptr_atomic_buffer_load_v4i32:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: .LBB8_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_load_b128 v[1:4], off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v4, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v4i32:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
; GFX12-SDAG-TRUE16-NEXT: global_wb
; GFX12-SDAG-TRUE16-NEXT: v_nop
; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-SDAG-TRUE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: .LBB8_1: ; %bb1
; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b128 v[2:5], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX12-SDAG-TRUE16-NEXT: s_endpgm
;
-; GFX12-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i64:
+; GFX12-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v4i32:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
; GFX12-SDAG-FAKE16-NEXT: global_wb
; GFX12-SDAG-FAKE16-NEXT: v_nop
; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-SDAG-FAKE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: .LBB8_1: ; %bb1
; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b128 v[2:5], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX12-SDAG-FAKE16-NEXT: s_endpgm
;
-; GFX12-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i64:
+; GFX12-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v4i32:
; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
; GFX12-GISEL-TRUE16-NEXT: global_wb
; GFX12-GISEL-TRUE16-NEXT: v_nop
; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-GISEL-TRUE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: .LBB8_1: ; %bb1
; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b128 v[2:5], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX12-GISEL-TRUE16-NEXT: s_endpgm
;
-; GFX12-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i64:
+; GFX12-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v4i32:
; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
; GFX12-GISEL-FAKE16-NEXT: global_wb
; GFX12-GISEL-FAKE16-NEXT: v_nop
; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-GISEL-FAKE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: .LBB8_1: ; %bb1
; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b128 v[2:5], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX12-GISEL-FAKE16-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- %id.zext = zext i32 %id to i64
- br label %bb1
-bb1:
- %load = call i64 @llvm.amdgcn.raw.ptr.atomic.buffer.load.i64(ptr addrspace(8) %ptr, i32 4, i32 0, i32 1)
- %cmp = icmp eq i64 %load, %id.zext
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-
-define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v2i16(ptr addrspace(8) %ptr) {
-; GFX11-LABEL: raw_ptr_atomic_buffer_load_v2i16:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB6_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b32 v1, off, s[0:3], 0 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB6_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: raw_ptr_atomic_buffer_load_v2i16:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB6_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB6_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
-; GFX12-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v2i16:
+; GFX12-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v4i32:
; GFX12-TRUE16: ; %bb.0: ; %bb
; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-TRUE16-NEXT: .LBB8_1: ; %bb1
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-TRUE16-NEXT: buffer_load_b128 v[2:5], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX12-TRUE16-NEXT: s_endpgm
-; GFX12-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v2i16:
+; GFX12-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v4i32:
; GFX12-FAKE16: ; %bb.0: ; %bb
; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-FAKE16-NEXT: .LBB8_1: ; %bb1
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-FAKE16-NEXT: buffer_load_b128 v[2:5], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX12-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX12-FAKE16-NEXT: s_endpgm
bb:
%id = tail call i32 @llvm.amdgcn.workitem.id.x()
br label %bb1
bb1:
- %load = call <2 x i16> @llvm.amdgcn.raw.ptr.atomic.buffer.load.v2i16(ptr addrspace(8) %ptr, i32 0, i32 0, i32 1)
- %bitcast = bitcast <2 x i16> %load to i32
- %cmp = icmp eq i32 %bitcast, %id
+ %load = call <4 x i32> @llvm.amdgcn.raw.ptr.atomic.buffer.load.v4i32(ptr addrspace(8) %ptr, i32 4, i32 0, i32 1)
+ %extracted = extractelement <4 x i32> %load, i32 3
+ %cmp = icmp eq i32 %extracted, %id
br i1 %cmp, label %bb1, label %bb2
bb2:
ret void
}
-define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v4i16(ptr addrspace(8) %ptr) {
-; GFX11-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+define amdgpu_kernel void @raw_ptr_atomic_buffer_load_ptr(ptr addrspace(8) %ptr) {
+; GFX11-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_ptr:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-SDAG-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: .LBB9_1: ; %bb1
; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: flat_load_b32 v1, v[1:2]
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX11-SDAG-TRUE16-NEXT: s_endpgm
;
-; GFX11-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX11-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_load_ptr:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-SDAG-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: .LBB9_1: ; %bb1
; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX11-SDAG-FAKE16-NEXT: flat_load_b32 v1, v[1:2]
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX11-SDAG-FAKE16-NEXT: s_endpgm
;
-; GFX11-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX11-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_ptr:
; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-GISEL-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: .LBB9_1: ; %bb1
; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-TRUE16-NEXT: buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX11-GISEL-TRUE16-NEXT: flat_load_b32 v1, v[1:2]
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX11-GISEL-TRUE16-NEXT: s_endpgm
;
-; GFX11-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX11-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_load_ptr:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-GISEL-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: .LBB9_1: ; %bb1
; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-FAKE16-NEXT: buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX11-GISEL-FAKE16-NEXT: flat_load_b32 v1, v[1:2]
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX11-GISEL-FAKE16-NEXT: s_endpgm
;
-; GFX11-GISEL-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX11-GISEL-LABEL: raw_ptr_atomic_buffer_load_ptr:
; GFX11-GISEL: ; %bb.0: ; %bb
; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
-; GFX11-GISEL-NEXT: .LBB7_1: ; %bb1
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: .LBB9_1: ; %bb1
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-GISEL-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-GISEL-NEXT: s_pack_ll_b32_b16 s5, s5, s6
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX11-GISEL-NEXT: flat_load_b32 v1, v[1:2]
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
; GFX11-GISEL-NEXT: s_endpgm
;
-; GFX12-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX12-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_ptr:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
; GFX12-SDAG-TRUE16-NEXT: global_wb
; GFX12-SDAG-TRUE16-NEXT: v_nop
@@ -785,21 +2326,22 @@ define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v4i16(ptr addrspace(8) %pt
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-SDAG-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: .LBB9_1: ; %bb1
; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-TRUE16-NEXT: flat_load_b32 v1, v[2:3]
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX12-SDAG-TRUE16-NEXT: s_endpgm
;
-; GFX12-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX12-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_load_ptr:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
; GFX12-SDAG-FAKE16-NEXT: global_wb
; GFX12-SDAG-FAKE16-NEXT: v_nop
@@ -808,23 +2350,22 @@ define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v4i16(ptr addrspace(8) %pt
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-SDAG-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: .LBB9_1: ; %bb1
; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX12-SDAG-FAKE16-NEXT: flat_load_b32 v1, v[2:3]
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX12-SDAG-FAKE16-NEXT: s_endpgm
;
-; GFX12-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX12-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_ptr:
; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
; GFX12-GISEL-TRUE16-NEXT: global_wb
; GFX12-GISEL-TRUE16-NEXT: v_nop
@@ -833,23 +2374,26 @@ define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v4i16(ptr addrspace(8) %pt
; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-GISEL-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: .LBB9_1: ; %bb1
; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s5, v2
-; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s6, v3
-; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX12-GISEL-TRUE16-NEXT: flat_load_b32 v1, v[2:3]
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX12-GISEL-TRUE16-NEXT: s_endpgm
;
-; GFX12-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX12-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_load_ptr:
; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
; GFX12-GISEL-FAKE16-NEXT: global_wb
; GFX12-GISEL-FAKE16-NEXT: v_nop
@@ -858,168 +2402,24 @@ define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v4i16(ptr addrspace(8) %pt
; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-GISEL-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: .LBB9_1: ; %bb1
; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s5, v2
-; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s6, v3
-; GFX12-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX12-GISEL-FAKE16-NEXT: flat_load_b32 v1, v[2:3]
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX12-GISEL-FAKE16-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- br label %bb1
-bb1:
- %load = call <4 x i16> @llvm.amdgcn.raw.ptr.atomic.buffer.load.v4i16(ptr addrspace(8) %ptr, i32 4, i32 0, i32 1)
- %shortened = shufflevector <4 x i16> %load, <4 x i16> poison, <2 x i32> <i32 0, i32 2>
- %bitcast = bitcast <2 x i16> %shortened to i32
- %cmp = icmp eq i32 %bitcast, %id
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-
-define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v4i32(ptr addrspace(8) %ptr) {
-; GFX11-LABEL: raw_ptr_atomic_buffer_load_v4i32:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB8_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b128 v[1:4], off, s[0:3], 0 offset:4 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v4, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB8_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: raw_ptr_atomic_buffer_load_v4i32:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB8_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: buffer_load_b128 v[2:5], off, s[0:3], null offset:4 th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB8_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
-; GFX12-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v4i32:
-; GFX12-TRUE16: ; %bb.0: ; %bb
-; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-TRUE16-NEXT: .LBB8_1: ; %bb1
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b128 v[2:5], off, s[0:3], null offset:4 th:TH_LOAD_NT
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
-; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %bb2
-; GFX12-TRUE16-NEXT: s_endpgm
-; GFX12-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v4i32:
-; GFX12-FAKE16: ; %bb.0: ; %bb
-; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-FAKE16-NEXT: .LBB8_1: ; %bb1
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b128 v[2:5], off, s[0:3], null offset:4 th:TH_LOAD_NT
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
-; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %bb2
-; GFX12-FAKE16-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- br label %bb1
-bb1:
- %load = call <4 x i32> @llvm.amdgcn.raw.ptr.atomic.buffer.load.v4i32(ptr addrspace(8) %ptr, i32 4, i32 0, i32 1)
- %extracted = extractelement <4 x i32> %load, i32 3
- %cmp = icmp eq i32 %extracted, %id
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-
-define amdgpu_kernel void @raw_ptr_atomic_buffer_load_ptr(ptr addrspace(8) %ptr) {
-; GFX11-LABEL: raw_ptr_atomic_buffer_load_ptr:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB9_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: flat_load_b32 v1, v[1:2]
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB9_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: raw_ptr_atomic_buffer_load_ptr:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB9_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: flat_load_b32 v1, v[2:3]
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB9_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
; GFX12-TRUE16-LABEL: raw_ptr_atomic_buffer_load_ptr:
; GFX12-TRUE16: ; %bb.0: ; %bb
; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
@@ -1083,3 +2483,6 @@ declare <4 x i32> @llvm.amdgcn.raw.ptr.atom.buffer.load.v4i32(ptr addrspace(8),
declare ptr @llvm.amdgcn.raw.ptr.atom.buffer.load.ptr(ptr addrspace(8), i32, i32, i32 immarg)
declare i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8), i32, i32, i32 immarg)
declare i32 @llvm.amdgcn.workitem.id.x()
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11: {{.*}}
+; GFX12: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.atomic.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.atomic.buffer.load.ll
index 736fc5bbdef91..cab204303ba66 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.atomic.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.atomic.buffer.load.ll
@@ -11,51 +11,230 @@
; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck %s -check-prefixes=GFX12,GFX12-GISEL-FAKE16
define amdgpu_kernel void @struct_atomic_buffer_load_i32(<4 x i32> %addr, i32 %index) {
-; GFX11-LABEL: struct_atomic_buffer_load_i32:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: .LBB0_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB0_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: struct_atomic_buffer_load_i32:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: .LBB0_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB0_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
+; GFX11-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_i32:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-TRUE16-NEXT: .LBB0_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_i32:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-FAKE16-NEXT: .LBB0_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_i32:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-TRUE16-NEXT: .LBB0_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_i32:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-FAKE16-NEXT: .LBB0_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: struct_atomic_buffer_load_i32:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-NEXT: .LBB0_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_i32:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-TRUE16-NEXT: .LBB0_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_i32:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-FAKE16-NEXT: .LBB0_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_i32:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-TRUE16-NEXT: .LBB0_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_i32:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-FAKE16-NEXT: .LBB0_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
bb:
%id = tail call i32 @llvm.amdgcn.workitem.id.x()
br label %bb1
@@ -68,309 +247,2027 @@ bb2:
}
define amdgpu_kernel void @struct_atomic_buffer_load_i32_const_idx(<4 x i32> %addr) {
-; GFX11-LABEL: struct_atomic_buffer_load_i32_const_idx:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB1_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB1_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: struct_atomic_buffer_load_i32_const_idx:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: v_mov_b32_e32 v1, 15
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB1_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB1_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- br label %bb1
-bb1:
- %load = call i32 @llvm.amdgcn.struct.atomic.buffer.load.i32(<4 x i32> %addr, i32 15, i32 0, i32 0, i32 1)
- %cmp = icmp eq i32 %load, %id
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-
-define amdgpu_kernel void @struct_atomic_buffer_load_i32_off(<4 x i32> %addr, i32 %index) {
-; GFX11-LABEL: struct_atomic_buffer_load_i32_off:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: .LBB2_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB2_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: struct_atomic_buffer_load_i32_off:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: .LBB2_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB2_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- br label %bb1
-bb1:
- %load = call i32 @llvm.amdgcn.struct.atomic.buffer.load.i32(<4 x i32> %addr, i32 %index, i32 0, i32 0, i32 1)
- %cmp = icmp eq i32 %load, %id
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-
-define amdgpu_kernel void @struct_atomic_buffer_load_i32_soff(<4 x i32> %addr, i32 %index) {
-; GFX11-LABEL: struct_atomic_buffer_load_i32_soff:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: .LBB3_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 4 idxen offset:4 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB3_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: struct_atomic_buffer_load_i32_soff:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: s_mov_b32 s5, 4
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: .LBB3_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: buffer_load_b32 v2, v1, s[0:3], s5 idxen offset:4 th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB3_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- br label %bb1
-bb1:
- %load = call i32 @llvm.amdgcn.struct.atomic.buffer.load.i32(<4 x i32> %addr, i32 %index, i32 4, i32 4, i32 1)
- %cmp = icmp eq i32 %load, %id
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-define amdgpu_kernel void @struct_atomic_buffer_load_i32_dlc(<4 x i32> %addr, i32 %index) {
-; GFX11-LABEL: struct_atomic_buffer_load_i32_dlc:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: .LBB4_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen offset:4 dlc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB4_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: struct_atomic_buffer_load_i32_dlc:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: .LBB4_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT_RT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB4_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- br label %bb1
-bb1:
- %load = call i32 @llvm.amdgcn.struct.atomic.buffer.load.i32(<4 x i32> %addr, i32 %index, i32 4, i32 0, i32 4)
- %cmp = icmp eq i32 %load, %id
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-
-define amdgpu_kernel void @struct_nonatomic_buffer_load_i32(<4 x i32> %addr, i32 %index) {
-; GFX11-LABEL: struct_nonatomic_buffer_load_i32:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 idxen offset:4 glc
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: .LBB5_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_and_b32 s1, exec_lo, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s1, s0
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB5_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: struct_nonatomic_buffer_load_i32:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-NEXT: .LBB5_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_and_b32 s1, exec_lo, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_or_b32 s0, s1, s0
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB5_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
+; GFX11-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_i32_const_idx:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: .LBB1_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_i32_const_idx:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: .LBB1_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_i32_const_idx:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: .LBB1_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_i32_const_idx:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: .LBB1_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: struct_atomic_buffer_load_i32_const_idx:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: .LBB1_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB1_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_i32_const_idx:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 15
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: .LBB1_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_i32_const_idx:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 15
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: .LBB1_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_i32_const_idx:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 15
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: .LBB1_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_i32_const_idx:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 15
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: .LBB1_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ br label %bb1
+bb1:
+ %load = call i32 @llvm.amdgcn.struct.atomic.buffer.load.i32(<4 x i32> %addr, i32 15, i32 0, i32 0, i32 1)
+ %cmp = icmp eq i32 %load, %id
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+
+define amdgpu_kernel void @struct_atomic_buffer_load_i32_off(<4 x i32> %addr, i32 %index) {
+; GFX11-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_i32_off:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-TRUE16-NEXT: .LBB2_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_i32_off:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-FAKE16-NEXT: .LBB2_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_i32_off:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-TRUE16-NEXT: .LBB2_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_i32_off:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-FAKE16-NEXT: .LBB2_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: struct_atomic_buffer_load_i32_off:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-NEXT: .LBB2_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB2_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_i32_off:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-TRUE16-NEXT: .LBB2_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_i32_off:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-FAKE16-NEXT: .LBB2_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_i32_off:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-TRUE16-NEXT: .LBB2_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_i32_off:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-FAKE16-NEXT: .LBB2_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ br label %bb1
+bb1:
+ %load = call i32 @llvm.amdgcn.struct.atomic.buffer.load.i32(<4 x i32> %addr, i32 %index, i32 0, i32 0, i32 1)
+ %cmp = icmp eq i32 %load, %id
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+
+define amdgpu_kernel void @struct_atomic_buffer_load_i32_soff(<4 x i32> %addr, i32 %index) {
+; GFX11-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_i32_soff:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-TRUE16-NEXT: .LBB3_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 4 idxen offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_i32_soff:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-FAKE16-NEXT: .LBB3_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 4 idxen offset:4 glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_i32_soff:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-TRUE16-NEXT: .LBB3_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 4 idxen offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_i32_soff:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-FAKE16-NEXT: .LBB3_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 4 idxen offset:4 glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: struct_atomic_buffer_load_i32_soff:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-NEXT: .LBB3_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: buffer_load_b32 v2, v1, s[0:3], 4 idxen offset:4 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB3_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_i32_soff:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s5, 4
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-TRUE16-NEXT: .LBB3_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], s5 idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_i32_soff:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s5, 4
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-FAKE16-NEXT: .LBB3_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], s5 idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_i32_soff:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s5, 4
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX12-GISEL-TRUE16-NEXT: .LBB3_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], s5 idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s6, s6, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s7, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s6, s6, s7
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_i32_soff:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s5, 4
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX12-GISEL-FAKE16-NEXT: .LBB3_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], s5 idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s6, s6, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s7, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s6, s6, s7
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ br label %bb1
+bb1:
+ %load = call i32 @llvm.amdgcn.struct.atomic.buffer.load.i32(<4 x i32> %addr, i32 %index, i32 4, i32 4, i32 1)
+ %cmp = icmp eq i32 %load, %id
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+define amdgpu_kernel void @struct_atomic_buffer_load_i32_dlc(<4 x i32> %addr, i32 %index) {
+; GFX11-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_i32_dlc:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-TRUE16-NEXT: .LBB4_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen offset:4 dlc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_i32_dlc:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-FAKE16-NEXT: .LBB4_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen offset:4 dlc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_i32_dlc:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-TRUE16-NEXT: .LBB4_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen offset:4 dlc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_i32_dlc:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-FAKE16-NEXT: .LBB4_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen offset:4 dlc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: struct_atomic_buffer_load_i32_dlc:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-NEXT: .LBB4_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen offset:4 dlc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB4_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_i32_dlc:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-TRUE16-NEXT: .LBB4_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT_RT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_i32_dlc:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-FAKE16-NEXT: .LBB4_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT_RT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_i32_dlc:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-TRUE16-NEXT: .LBB4_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT_RT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_i32_dlc:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-FAKE16-NEXT: .LBB4_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT_RT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ br label %bb1
+bb1:
+ %load = call i32 @llvm.amdgcn.struct.atomic.buffer.load.i32(<4 x i32> %addr, i32 %index, i32 4, i32 0, i32 4)
+ %cmp = icmp eq i32 %load, %id
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+
+define amdgpu_kernel void @struct_nonatomic_buffer_load_i32(<4 x i32> %addr, i32 %index) {
+; GFX11-SDAG-TRUE16-LABEL: struct_nonatomic_buffer_load_i32:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: .LBB5_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s0, s1, s0
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: struct_nonatomic_buffer_load_i32:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-FAKE16-NEXT: .LBB5_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s0, s1, s0
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: struct_nonatomic_buffer_load_i32:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-TRUE16-NEXT: .LBB5_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s0, s2, s0
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s1, s1, s2
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: struct_nonatomic_buffer_load_i32:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-FAKE16-NEXT: .LBB5_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s0, s2, s0
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s1, s1, s2
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: struct_nonatomic_buffer_load_i32:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-NEXT: .LBB5_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s0, s2, s0
+; GFX11-GISEL-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX11-GISEL-NEXT: s_or_b32 s1, s1, s2
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB5_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: struct_nonatomic_buffer_load_i32:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v1, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-TRUE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s0, s1, s0
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: struct_nonatomic_buffer_load_i32:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v1, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-FAKE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s0, s1, s0
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: struct_nonatomic_buffer_load_i32:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v1, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-TRUE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s0, s2, s0
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s1, s1, s2
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: struct_nonatomic_buffer_load_i32:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v1, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr1
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-FAKE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s0, s2, s0
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s1, s1, s2
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ br label %bb1
+bb1:
+ %load = call i32 @llvm.amdgcn.struct.buffer.load.i32(<4 x i32> %addr, i32 %index, i32 4, i32 0, i32 1)
+ %cmp = icmp eq i32 %load, %id
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+
+define amdgpu_kernel void @struct_atomic_buffer_load_i64(<4 x i32> %addr, i32 %index) {
+; GFX11-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_i64:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, s6
+; GFX11-SDAG-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b64 v[3:4], v2, s[0:3], 0 idxen offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[3:4], v[0:1]
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_i64:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, s6
+; GFX11-SDAG-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b64 v[3:4], v2, s[0:3], 0 idxen offset:4 glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[3:4], v[0:1]
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_i64:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s6
+; GFX11-GISEL-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b64 v[3:4], v2, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[3:4], v[0:1]
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_i64:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s6
+; GFX11-GISEL-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b64 v[3:4], v2, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[3:4], v[0:1]
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: struct_atomic_buffer_load_i64:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, s6
+; GFX11-GISEL-NEXT: .LBB6_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: buffer_load_b64 v[3:4], v2, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[3:4], v[0:1]
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_i64:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, s6
+; GFX12-SDAG-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_i64:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, s6
+; GFX12-SDAG-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_i64:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s6
+; GFX12-GISEL-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_i64:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s6
+; GFX12-GISEL-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %id.zext = zext i32 %id to i64
+ br label %bb1
+bb1:
+ %load = call i64 @llvm.amdgcn.struct.atomic.buffer.load.i64(<4 x i32> %addr, i32 %index, i32 4, i32 0, i32 1)
+ %cmp = icmp eq i64 %load, %id.zext
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+
+define amdgpu_kernel void @struct_atomic_buffer_load_v2i16(<4 x i32> %addr, i32 %index) {
+; GFX11-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_v2i16:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_v2i16:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_v2i16:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_v2i16:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: struct_atomic_buffer_load_v2i16:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-NEXT: .LBB7_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_v2i16:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_v2i16:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_v2i16:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_v2i16:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ br label %bb1
+bb1:
+ %load = call <2 x i16> @llvm.amdgcn.struct.atomic.buffer.load.v2i16(<4 x i32> %addr, i32 %index, i32 0, i32 0, i32 1)
+ %bitcast = bitcast <2 x i16> %load to i32
+ %cmp = icmp eq i32 %bitcast, %id
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+
+define amdgpu_kernel void @struct_atomic_buffer_load_v4i16(<4 x i32> %addr, i32 %index) {
+; GFX11-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
+; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX11-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
+; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX11-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-NEXT: .LBB8_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_readfirstlane_b32 s6, v2
+; GFX11-GISEL-NEXT: v_readfirstlane_b32 s7, v3
+; GFX11-GISEL-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
+; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
+; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
bb:
%id = tail call i32 @llvm.amdgcn.workitem.id.x()
br label %bb1
bb1:
- %load = call i32 @llvm.amdgcn.struct.buffer.load.i32(<4 x i32> %addr, i32 %index, i32 4, i32 0, i32 1)
- %cmp = icmp eq i32 %load, %id
+ %load = call <4 x i16> @llvm.amdgcn.struct.atomic.buffer.load.v4i16(<4 x i32> %addr, i32 %index, i32 4, i32 0, i32 1)
+ %shortened = shufflevector <4 x i16> %load, <4 x i16> poison, <2 x i32> <i32 0, i32 2>
+ %bitcast = bitcast <2 x i16> %shortened to i32
+ %cmp = icmp eq i32 %bitcast, %id
br i1 %cmp, label %bb1, label %bb2
bb2:
ret void
}
-define amdgpu_kernel void @struct_atomic_buffer_load_i64(<4 x i32> %addr, i32 %index) {
-; GFX11-LABEL: struct_atomic_buffer_load_i64:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, s6
-; GFX11-NEXT: .LBB6_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: buffer_load_b64 v[3:4], v2, s[0:3], 0 idxen offset:4 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[3:4], v[0:1]
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB6_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
+define amdgpu_kernel void @struct_atomic_buffer_load_v4i32(<4 x i32> %addr, i32 %index) {
+; GFX11-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_v4i32:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-TRUE16-NEXT: .LBB9_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB9_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
;
-; GFX12-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_i64:
+; GFX11-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_v4i32:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-FAKE16-NEXT: .LBB9_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB9_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_v4i32:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-TRUE16-NEXT: .LBB9_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB9_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_v4i32:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-FAKE16-NEXT: .LBB9_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB9_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: struct_atomic_buffer_load_v4i32:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-NEXT: .LBB9_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB9_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_v4i32:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
; GFX12-SDAG-TRUE16-NEXT: global_wb
; GFX12-SDAG-TRUE16-NEXT: v_nop
@@ -378,25 +2275,24 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i64(<4 x i32> %addr, i32 %i
; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, s6
-; GFX12-SDAG-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-TRUE16-NEXT: .LBB9_1: ; %bb1
; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX12-SDAG-TRUE16-NEXT: s_endpgm
;
-; GFX12-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_i64:
+; GFX12-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_v4i32:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
; GFX12-SDAG-FAKE16-NEXT: global_wb
; GFX12-SDAG-FAKE16-NEXT: v_nop
@@ -404,25 +2300,24 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i64(<4 x i32> %addr, i32 %i
; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, s6
-; GFX12-SDAG-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-FAKE16-NEXT: .LBB9_1: ; %bb1
; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX12-SDAG-FAKE16-NEXT: s_endpgm
;
-; GFX12-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_i64:
+; GFX12-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_v4i32:
; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
; GFX12-GISEL-TRUE16-NEXT: global_wb
; GFX12-GISEL-TRUE16-NEXT: v_nop
@@ -431,24 +2326,27 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i64(<4 x i32> %addr, i32 %i
; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s6
-; GFX12-GISEL-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-TRUE16-NEXT: .LBB9_1: ; %bb1
; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX12-GISEL-TRUE16-NEXT: s_endpgm
;
-; GFX12-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_i64:
+; GFX12-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_v4i32:
; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
; GFX12-GISEL-FAKE16-NEXT: global_wb
; GFX12-GISEL-FAKE16-NEXT: v_nop
@@ -457,94 +2355,39 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i64(<4 x i32> %addr, i32 %i
; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s6
-; GFX12-GISEL-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-FAKE16-NEXT: .LBB9_1: ; %bb1
; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX12-GISEL-FAKE16-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- %id.zext = zext i32 %id to i64
- br label %bb1
-bb1:
- %load = call i64 @llvm.amdgcn.struct.atomic.buffer.load.i64(<4 x i32> %addr, i32 %index, i32 4, i32 0, i32 1)
- %cmp = icmp eq i64 %load, %id.zext
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-
-define amdgpu_kernel void @struct_atomic_buffer_load_v2i16(<4 x i32> %addr, i32 %index) {
-; GFX11-LABEL: struct_atomic_buffer_load_v2i16:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: .LBB7_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB7_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: struct_atomic_buffer_load_v2i16:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: .LBB7_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB7_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
bb:
%id = tail call i32 @llvm.amdgcn.workitem.id.x()
br label %bb1
bb1:
- %load = call <2 x i16> @llvm.amdgcn.struct.atomic.buffer.load.v2i16(<4 x i32> %addr, i32 %index, i32 0, i32 0, i32 1)
- %bitcast = bitcast <2 x i16> %load to i32
- %cmp = icmp eq i32 %bitcast, %id
+ %load = call <4 x i32> @llvm.amdgcn.struct.atomic.buffer.load.v4i32(<4 x i32> %addr, i32 %index, i32 4, i32 0, i32 1)
+ %extracted = extractelement <4 x i32> %load, i32 3
+ %cmp = icmp eq i32 %extracted, %id
br i1 %cmp, label %bb1, label %bb2
bb2:
ret void
}
-define amdgpu_kernel void @struct_atomic_buffer_load_v4i16(<4 x i32> %addr, i32 %index) {
-; GFX11-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_v4i16:
+define amdgpu_kernel void @struct_atomic_buffer_load_ptr(<4 x i32> %addr, i32 %index) {
+; GFX11-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_ptr:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
; GFX11-SDAG-TRUE16-NEXT: s_clause 0x1
; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
@@ -553,20 +2396,21 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v4i16(<4 x i32> %addr, i32
; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-SDAG-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: .LBB10_1: ; %bb1
; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: flat_load_b32 v2, v[2:3]
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX11-SDAG-TRUE16-NEXT: s_endpgm
;
-; GFX11-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX11-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_ptr:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
; GFX11-SDAG-FAKE16-NEXT: s_clause 0x1
; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
@@ -575,94 +2419,102 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v4i16(<4 x i32> %addr, i32
; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-SDAG-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: .LBB10_1: ; %bb1
; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX11-SDAG-FAKE16-NEXT: flat_load_b32 v2, v[2:3]
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX11-SDAG-FAKE16-NEXT: s_endpgm
;
-; GFX11-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX11-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_ptr:
; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-GISEL-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: .LBB10_1: ; %bb1
; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s5, v2
-; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s6, v3
-; GFX11-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX11-GISEL-TRUE16-NEXT: flat_load_b32 v2, v[2:3]
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX11-GISEL-TRUE16-NEXT: s_endpgm
;
-; GFX11-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX11-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_ptr:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-GISEL-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: .LBB10_1: ; %bb1
; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s5, v2
-; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s6, v3
-; GFX11-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX11-GISEL-FAKE16-NEXT: flat_load_b32 v2, v[2:3]
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX11-GISEL-FAKE16-NEXT: s_endpgm
;
-; GFX11-GISEL-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX11-GISEL-LABEL: struct_atomic_buffer_load_ptr:
; GFX11-GISEL: ; %bb.0: ; %bb
; GFX11-GISEL-NEXT: s_clause 0x1
; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-GISEL-NEXT: .LBB8_1: ; %bb1
+; GFX11-GISEL-NEXT: .LBB10_1: ; %bb1
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-NEXT: v_readfirstlane_b32 s5, v2
-; GFX11-GISEL-NEXT: v_readfirstlane_b32 s6, v3
-; GFX11-GISEL-NEXT: s_pack_ll_b32_b16 s5, s5, s6
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX11-GISEL-NEXT: flat_load_b32 v2, v[2:3]
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
; GFX11-GISEL-NEXT: s_endpgm
;
-; GFX12-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX12-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_ptr:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
; GFX12-SDAG-TRUE16-NEXT: global_wb
; GFX12-SDAG-TRUE16-NEXT: v_nop
@@ -675,20 +2527,21 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v4i16(<4 x i32> %addr, i32
; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-SDAG-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: .LBB10_1: ; %bb1
; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: flat_load_b32 v2, v[2:3]
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX12-SDAG-TRUE16-NEXT: s_endpgm
;
-; GFX12-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX12-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_ptr:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
; GFX12-SDAG-FAKE16-NEXT: global_wb
; GFX12-SDAG-FAKE16-NEXT: v_nop
@@ -701,22 +2554,21 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v4i16(<4 x i32> %addr, i32
; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-SDAG-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: .LBB10_1: ; %bb1
; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX12-SDAG-FAKE16-NEXT: flat_load_b32 v2, v[2:3]
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX12-SDAG-FAKE16-NEXT: s_endpgm
;
-; GFX12-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX12-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_ptr:
; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
; GFX12-GISEL-TRUE16-NEXT: global_wb
; GFX12-GISEL-TRUE16-NEXT: v_nop
@@ -727,24 +2579,27 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v4i16(<4 x i32> %addr, i32
; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-GISEL-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: .LBB10_1: ; %bb1
; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s5, v2
-; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s6, v3
-; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX12-GISEL-TRUE16-NEXT: flat_load_b32 v2, v[2:3]
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX12-GISEL-TRUE16-NEXT: s_endpgm
;
-; GFX12-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX12-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_ptr:
; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
; GFX12-GISEL-FAKE16-NEXT: global_wb
; GFX12-GISEL-FAKE16-NEXT: v_nop
@@ -755,143 +2610,25 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v4i16(<4 x i32> %addr, i32
; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-GISEL-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: .LBB10_1: ; %bb1
; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s5, v2
-; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s6, v3
-; GFX12-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX12-GISEL-FAKE16-NEXT: flat_load_b32 v2, v[2:3]
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX12-GISEL-FAKE16-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- br label %bb1
-bb1:
- %load = call <4 x i16> @llvm.amdgcn.struct.atomic.buffer.load.v4i16(<4 x i32> %addr, i32 %index, i32 4, i32 0, i32 1)
- %shortened = shufflevector <4 x i16> %load, <4 x i16> poison, <2 x i32> <i32 0, i32 2>
- %bitcast = bitcast <2 x i16> %shortened to i32
- %cmp = icmp eq i32 %bitcast, %id
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-
-define amdgpu_kernel void @struct_atomic_buffer_load_v4i32(<4 x i32> %addr, i32 %index) {
-; GFX11-LABEL: struct_atomic_buffer_load_v4i32:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: .LBB9_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], 0 idxen offset:4 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB9_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: struct_atomic_buffer_load_v4i32:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: .LBB9_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB9_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- br label %bb1
-bb1:
- %load = call <4 x i32> @llvm.amdgcn.struct.atomic.buffer.load.v4i32(<4 x i32> %addr, i32 %index, i32 4, i32 0, i32 1)
- %extracted = extractelement <4 x i32> %load, i32 3
- %cmp = icmp eq i32 %extracted, %id
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-
-define amdgpu_kernel void @struct_atomic_buffer_load_ptr(<4 x i32> %addr, i32 %index) {
-; GFX11-LABEL: struct_atomic_buffer_load_ptr:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: .LBB10_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: flat_load_b32 v2, v[2:3]
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB10_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: struct_atomic_buffer_load_ptr:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: .LBB10_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: flat_load_b32 v2, v[2:3]
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB10_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
bb:
%id = tail call i32 @llvm.amdgcn.workitem.id.x()
br label %bb1
@@ -912,3 +2649,6 @@ declare <4 x i32> @llvm.amdgcn.struct.atom.buffer.load.v4i32(<4 x i32>, i32, i32
declare ptr @llvm.amdgcn.struct.atom.buffer.load.ptr(<4 x i32>, i32, i32, i32, i32 immarg)
declare i32 @llvm.amdgcn.struct.buffer.load.i32(<4 x i32>, i32, i32, i32, i32 immarg)
declare i32 @llvm.amdgcn.workitem.id.x()
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11: {{.*}}
+; GFX12: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.atomic.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.atomic.buffer.load.ll
index d3ba99275f7f6..51c8943738788 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.atomic.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.atomic.buffer.load.ll
@@ -11,51 +11,230 @@
; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck %s -check-prefixes=GFX12,GFX12-GISEL-FAKE16
define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32(ptr addrspace(8) %ptr, i32 %index) {
-; GFX11-LABEL: struct_ptr_atomic_buffer_load_i32:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: .LBB0_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB0_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: struct_ptr_atomic_buffer_load_i32:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: .LBB0_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB0_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
+; GFX11-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i32:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-TRUE16-NEXT: .LBB0_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i32:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-FAKE16-NEXT: .LBB0_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i32:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-TRUE16-NEXT: .LBB0_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i32:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-FAKE16-NEXT: .LBB0_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: struct_ptr_atomic_buffer_load_i32:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-NEXT: .LBB0_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i32:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-TRUE16-NEXT: .LBB0_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i32:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-FAKE16-NEXT: .LBB0_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i32:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-TRUE16-NEXT: .LBB0_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i32:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-FAKE16-NEXT: .LBB0_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
bb:
%id = tail call i32 @llvm.amdgcn.workitem.id.x()
br label %bb1
@@ -68,309 +247,2027 @@ bb2:
}
define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32_const_idx(ptr addrspace(8) %ptr) {
-; GFX11-LABEL: struct_ptr_atomic_buffer_load_i32_const_idx:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .LBB1_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB1_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: struct_ptr_atomic_buffer_load_i32_const_idx:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: v_mov_b32_e32 v1, 15
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB1_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB1_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- br label %bb1
-bb1:
- %load = call i32 @llvm.amdgcn.struct.ptr.atomic.buffer.load.i32(ptr addrspace(8) %ptr, i32 15, i32 0, i32 0, i32 1)
- %cmp = icmp eq i32 %load, %id
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-
-define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32_off(ptr addrspace(8) %ptr, i32 %index) {
-; GFX11-LABEL: struct_ptr_atomic_buffer_load_i32_off:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: .LBB2_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB2_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: struct_ptr_atomic_buffer_load_i32_off:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: .LBB2_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB2_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- br label %bb1
-bb1:
- %load = call i32 @llvm.amdgcn.struct.ptr.atomic.buffer.load.i32(ptr addrspace(8) %ptr, i32 %index, i32 0, i32 0, i32 1)
- %cmp = icmp eq i32 %load, %id
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-
-define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32_soff(ptr addrspace(8) %ptr, i32 %index) {
-; GFX11-LABEL: struct_ptr_atomic_buffer_load_i32_soff:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: .LBB3_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 4 idxen offset:4 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB3_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: struct_ptr_atomic_buffer_load_i32_soff:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: s_mov_b32 s5, 4
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: .LBB3_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: buffer_load_b32 v2, v1, s[0:3], s5 idxen offset:4 th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB3_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- br label %bb1
-bb1:
- %load = call i32 @llvm.amdgcn.struct.ptr.atomic.buffer.load.i32(ptr addrspace(8) %ptr, i32 %index, i32 4, i32 4, i32 1)
- %cmp = icmp eq i32 %load, %id
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32_dlc(ptr addrspace(8) %ptr, i32 %index) {
-; GFX11-LABEL: struct_ptr_atomic_buffer_load_i32_dlc:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: .LBB4_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen offset:4 dlc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB4_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: struct_ptr_atomic_buffer_load_i32_dlc:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: .LBB4_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT_RT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB4_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- br label %bb1
-bb1:
- %load = call i32 @llvm.amdgcn.struct.ptr.atomic.buffer.load.i32(ptr addrspace(8) %ptr, i32 %index, i32 4, i32 0, i32 4)
- %cmp = icmp eq i32 %load, %id
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-
-define amdgpu_kernel void @struct_ptr_nonatomic_buffer_load_i32(ptr addrspace(8) %ptr, i32 %index) {
-; GFX11-LABEL: struct_ptr_nonatomic_buffer_load_i32:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 idxen offset:4 glc
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: .LBB5_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_and_b32 s1, exec_lo, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s1, s0
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB5_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: struct_ptr_nonatomic_buffer_load_i32:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX12-NEXT: .LBB5_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_and_b32 s1, exec_lo, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_or_b32 s0, s1, s0
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB5_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
+; GFX11-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i32_const_idx:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: .LBB1_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i32_const_idx:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: .LBB1_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i32_const_idx:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: .LBB1_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i32_const_idx:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: .LBB1_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: struct_ptr_atomic_buffer_load_i32_const_idx:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: .LBB1_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB1_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i32_const_idx:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 15
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: .LBB1_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i32_const_idx:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 15
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: .LBB1_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i32_const_idx:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 15
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: .LBB1_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i32_const_idx:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 15
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: .LBB1_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ br label %bb1
+bb1:
+ %load = call i32 @llvm.amdgcn.struct.ptr.atomic.buffer.load.i32(ptr addrspace(8) %ptr, i32 15, i32 0, i32 0, i32 1)
+ %cmp = icmp eq i32 %load, %id
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+
+define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32_off(ptr addrspace(8) %ptr, i32 %index) {
+; GFX11-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i32_off:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-TRUE16-NEXT: .LBB2_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i32_off:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-FAKE16-NEXT: .LBB2_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i32_off:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-TRUE16-NEXT: .LBB2_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i32_off:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-FAKE16-NEXT: .LBB2_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: struct_ptr_atomic_buffer_load_i32_off:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-NEXT: .LBB2_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB2_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i32_off:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-TRUE16-NEXT: .LBB2_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i32_off:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-FAKE16-NEXT: .LBB2_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i32_off:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-TRUE16-NEXT: .LBB2_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i32_off:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-FAKE16-NEXT: .LBB2_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB2_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ br label %bb1
+bb1:
+ %load = call i32 @llvm.amdgcn.struct.ptr.atomic.buffer.load.i32(ptr addrspace(8) %ptr, i32 %index, i32 0, i32 0, i32 1)
+ %cmp = icmp eq i32 %load, %id
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+
+define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32_soff(ptr addrspace(8) %ptr, i32 %index) {
+; GFX11-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i32_soff:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-TRUE16-NEXT: .LBB3_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 4 idxen offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i32_soff:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-FAKE16-NEXT: .LBB3_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 4 idxen offset:4 glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i32_soff:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-TRUE16-NEXT: .LBB3_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 4 idxen offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i32_soff:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-FAKE16-NEXT: .LBB3_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 4 idxen offset:4 glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: struct_ptr_atomic_buffer_load_i32_soff:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-NEXT: .LBB3_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: buffer_load_b32 v2, v1, s[0:3], 4 idxen offset:4 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB3_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i32_soff:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s5, 4
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-TRUE16-NEXT: .LBB3_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], s5 idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i32_soff:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s5, 4
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-FAKE16-NEXT: .LBB3_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], s5 idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i32_soff:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s5, 4
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX12-GISEL-TRUE16-NEXT: .LBB3_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], s5 idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s6, s6, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s7, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s6, s6, s7
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i32_soff:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s5, 4
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX12-GISEL-FAKE16-NEXT: .LBB3_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], s5 idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s6, s6, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s7, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s6, s6, s7
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ br label %bb1
+bb1:
+ %load = call i32 @llvm.amdgcn.struct.ptr.atomic.buffer.load.i32(ptr addrspace(8) %ptr, i32 %index, i32 4, i32 4, i32 1)
+ %cmp = icmp eq i32 %load, %id
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32_dlc(ptr addrspace(8) %ptr, i32 %index) {
+; GFX11-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i32_dlc:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-TRUE16-NEXT: .LBB4_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen offset:4 dlc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i32_dlc:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-FAKE16-NEXT: .LBB4_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen offset:4 dlc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i32_dlc:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-TRUE16-NEXT: .LBB4_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen offset:4 dlc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i32_dlc:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-FAKE16-NEXT: .LBB4_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen offset:4 dlc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: struct_ptr_atomic_buffer_load_i32_dlc:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-NEXT: .LBB4_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen offset:4 dlc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB4_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i32_dlc:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-TRUE16-NEXT: .LBB4_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT_RT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i32_dlc:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-FAKE16-NEXT: .LBB4_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT_RT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i32_dlc:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-TRUE16-NEXT: .LBB4_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT_RT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i32_dlc:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-FAKE16-NEXT: .LBB4_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT_RT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB4_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ br label %bb1
+bb1:
+ %load = call i32 @llvm.amdgcn.struct.ptr.atomic.buffer.load.i32(ptr addrspace(8) %ptr, i32 %index, i32 4, i32 0, i32 4)
+ %cmp = icmp eq i32 %load, %id
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+
+define amdgpu_kernel void @struct_ptr_nonatomic_buffer_load_i32(ptr addrspace(8) %ptr, i32 %index) {
+; GFX11-SDAG-TRUE16-LABEL: struct_ptr_nonatomic_buffer_load_i32:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: .LBB5_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s0, s1, s0
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: struct_ptr_nonatomic_buffer_load_i32:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-FAKE16-NEXT: .LBB5_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s0, s1, s0
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: struct_ptr_nonatomic_buffer_load_i32:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-TRUE16-NEXT: .LBB5_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s0, s2, s0
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s1, s1, s2
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: struct_ptr_nonatomic_buffer_load_i32:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr1
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-FAKE16-NEXT: .LBB5_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s0, s2, s0
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s1, s1, s2
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: struct_ptr_nonatomic_buffer_load_i32:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-NEXT: .LBB5_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s0, s2, s0
+; GFX11-GISEL-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX11-GISEL-NEXT: s_or_b32 s1, s1, s2
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB5_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: struct_ptr_nonatomic_buffer_load_i32:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v1, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-TRUE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s0, s1, s0
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: struct_ptr_nonatomic_buffer_load_i32:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v1, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-FAKE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: s_and_b32 s1, exec_lo, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s0, s1, s0
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: struct_ptr_nonatomic_buffer_load_i32:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v1, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-TRUE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s0, s2, s0
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s1, s1, s2
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: struct_ptr_nonatomic_buffer_load_i32:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v1, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr1
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-GISEL-FAKE16-NEXT: .LBB5_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s2, exec_lo, vcc_lo
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s0, s2, s0
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s1, s1, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s2, exec_lo, s0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s1, s1, s2
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB5_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ br label %bb1
+bb1:
+ %load = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) %ptr, i32 %index, i32 4, i32 0, i32 1)
+ %cmp = icmp eq i32 %load, %id
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+
+define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i64(ptr addrspace(8) %ptr, i32 %index) {
+; GFX11-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i64:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, s6
+; GFX11-SDAG-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b64 v[3:4], v2, s[0:3], 0 idxen offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[3:4], v[0:1]
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i64:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, s6
+; GFX11-SDAG-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b64 v[3:4], v2, s[0:3], 0 idxen offset:4 glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[3:4], v[0:1]
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i64:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s6
+; GFX11-GISEL-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b64 v[3:4], v2, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[3:4], v[0:1]
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i64:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s6
+; GFX11-GISEL-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b64 v[3:4], v2, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[3:4], v[0:1]
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: struct_ptr_atomic_buffer_load_i64:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, s6
+; GFX11-GISEL-NEXT: .LBB6_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: buffer_load_b64 v[3:4], v2, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[3:4], v[0:1]
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB6_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i64:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, s6
+; GFX12-SDAG-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i64:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, s6
+; GFX12-SDAG-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i64:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s6
+; GFX12-GISEL-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i64:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s6
+; GFX12-GISEL-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %id.zext = zext i32 %id to i64
+ br label %bb1
+bb1:
+ %load = call i64 @llvm.amdgcn.struct.ptr.atomic.buffer.load.i64(ptr addrspace(8) %ptr, i32 %index, i32 4, i32 0, i32 1)
+ %cmp = icmp eq i64 %load, %id.zext
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+
+define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v2i16(ptr addrspace(8) %ptr, i32 %index) {
+; GFX11-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v2i16:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v2i16:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v2i16:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v2i16:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: struct_ptr_atomic_buffer_load_v2i16:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-NEXT: .LBB7_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB7_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v2i16:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v2i16:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v2i16:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-TRUE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v2i16:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-FAKE16-NEXT: .LBB7_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+bb:
+ %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+ br label %bb1
+bb1:
+ %load = call <2 x i16> @llvm.amdgcn.struct.ptr.atomic.buffer.load.v2i16(ptr addrspace(8) %ptr, i32 %index, i32 0, i32 0, i32 1)
+ %bitcast = bitcast <2 x i16> %load to i32
+ %cmp = icmp eq i32 %bitcast, %id
+ br i1 %cmp, label %bb1, label %bb2
+bb2:
+ ret void
+}
+
+define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i16(ptr addrspace(8) %ptr, i32 %index) {
+; GFX11-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX11-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
+; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX11-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
+; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX11-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-NEXT: .LBB8_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_readfirstlane_b32 s6, v2
+; GFX11-GISEL-NEXT: v_readfirstlane_b32 s7, v3
+; GFX11-GISEL-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
+; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT: s_endpgm
+;
+; GFX12-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
+; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
+; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
+; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
+; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
+; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX12-GISEL-FAKE16-NEXT: s_endpgm
bb:
%id = tail call i32 @llvm.amdgcn.workitem.id.x()
br label %bb1
bb1:
- %load = call i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8) %ptr, i32 %index, i32 4, i32 0, i32 1)
- %cmp = icmp eq i32 %load, %id
+ %load = call <4 x i16> @llvm.amdgcn.struct.ptr.atomic.buffer.load.v4i16(ptr addrspace(8) %ptr, i32 %index, i32 4, i32 0, i32 1)
+ %shortened = shufflevector <4 x i16> %load, <4 x i16> poison, <2 x i32> <i32 0, i32 2>
+ %bitcast = bitcast <2 x i16> %shortened to i32
+ %cmp = icmp eq i32 %bitcast, %id
br i1 %cmp, label %bb1, label %bb2
bb2:
ret void
}
-define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i64(ptr addrspace(8) %ptr, i32 %index) {
-; GFX11-LABEL: struct_ptr_atomic_buffer_load_i64:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, s6
-; GFX11-NEXT: .LBB6_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: buffer_load_b64 v[3:4], v2, s[0:3], 0 idxen offset:4 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[3:4], v[0:1]
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB6_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
+define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i32(ptr addrspace(8) %ptr, i32 %index) {
+; GFX11-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v4i32:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-TRUE16-NEXT: .LBB9_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB9_1
+; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT: s_endpgm
;
-; GFX12-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i64:
+; GFX11-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v4i32:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-SDAG-FAKE16-NEXT: .LBB9_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB9_1
+; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v4i32:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-TRUE16-NEXT: .LBB9_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB9_1
+; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v4i32:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-FAKE16-NEXT: .LBB9_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB9_1
+; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: struct_ptr_atomic_buffer_load_v4i32:
+; GFX11-GISEL: ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-GISEL-NEXT: .LBB9_1: ; %bb1
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB9_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v4i32:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
; GFX12-SDAG-TRUE16-NEXT: global_wb
; GFX12-SDAG-TRUE16-NEXT: v_nop
@@ -378,25 +2275,24 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i64(ptr addrspace(8) %p
; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, s6
-; GFX12-SDAG-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-TRUE16-NEXT: .LBB9_1: ; %bb1
; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX12-SDAG-TRUE16-NEXT: s_endpgm
;
-; GFX12-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i64:
+; GFX12-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v4i32:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
; GFX12-SDAG-FAKE16-NEXT: global_wb
; GFX12-SDAG-FAKE16-NEXT: v_nop
@@ -404,25 +2300,24 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i64(ptr addrspace(8) %p
; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, s6
-; GFX12-SDAG-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-FAKE16-NEXT: .LBB9_1: ; %bb1
; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX12-SDAG-FAKE16-NEXT: s_endpgm
;
-; GFX12-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i64:
+; GFX12-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v4i32:
; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
; GFX12-GISEL-TRUE16-NEXT: global_wb
; GFX12-GISEL-TRUE16-NEXT: v_nop
@@ -431,24 +2326,27 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i64(ptr addrspace(8) %p
; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s6
-; GFX12-GISEL-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-TRUE16-NEXT: .LBB9_1: ; %bb1
; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX12-GISEL-TRUE16-NEXT: s_endpgm
;
-; GFX12-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i64:
+; GFX12-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v4i32:
; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
; GFX12-GISEL-FAKE16-NEXT: global_wb
; GFX12-GISEL-FAKE16-NEXT: v_nop
@@ -457,94 +2355,39 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i64(ptr addrspace(8) %p
; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s6
-; GFX12-GISEL-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-FAKE16-NEXT: .LBB9_1: ; %bb1
; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB6_1
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX12-GISEL-FAKE16-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- %id.zext = zext i32 %id to i64
- br label %bb1
-bb1:
- %load = call i64 @llvm.amdgcn.struct.ptr.atomic.buffer.load.i64(ptr addrspace(8) %ptr, i32 %index, i32 4, i32 0, i32 1)
- %cmp = icmp eq i64 %load, %id.zext
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-
-define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v2i16(ptr addrspace(8) %ptr, i32 %index) {
-; GFX11-LABEL: struct_ptr_atomic_buffer_load_v2i16:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: .LBB7_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB7_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: struct_ptr_atomic_buffer_load_v2i16:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: .LBB7_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB7_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
bb:
%id = tail call i32 @llvm.amdgcn.workitem.id.x()
br label %bb1
bb1:
- %load = call <2 x i16> @llvm.amdgcn.struct.ptr.atomic.buffer.load.v2i16(ptr addrspace(8) %ptr, i32 %index, i32 0, i32 0, i32 1)
- %bitcast = bitcast <2 x i16> %load to i32
- %cmp = icmp eq i32 %bitcast, %id
+ %load = call <4 x i32> @llvm.amdgcn.struct.ptr.atomic.buffer.load.v4i32(ptr addrspace(8) %ptr, i32 %index, i32 4, i32 0, i32 1)
+ %extracted = extractelement <4 x i32> %load, i32 3
+ %cmp = icmp eq i32 %extracted, %id
br i1 %cmp, label %bb1, label %bb2
bb2:
ret void
}
-define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i16(ptr addrspace(8) %ptr, i32 %index) {
-; GFX11-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+define amdgpu_kernel void @struct_ptr_atomic_buffer_load_ptr(ptr addrspace(8) %ptr, i32 %index) {
+; GFX11-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_ptr:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
; GFX11-SDAG-TRUE16-NEXT: s_clause 0x1
; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
@@ -553,20 +2396,21 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i16(ptr addrspace(8)
; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-SDAG-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT: .LBB10_1: ; %bb1
; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: flat_load_b32 v2, v[2:3]
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX11-SDAG-TRUE16-NEXT: s_endpgm
;
-; GFX11-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX11-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_ptr:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
; GFX11-SDAG-FAKE16-NEXT: s_clause 0x1
; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
@@ -575,94 +2419,102 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i16(ptr addrspace(8)
; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-SDAG-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-SDAG-FAKE16-NEXT: .LBB10_1: ; %bb1
; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX11-SDAG-FAKE16-NEXT: flat_load_b32 v2, v[2:3]
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX11-SDAG-FAKE16-NEXT: s_endpgm
;
-; GFX11-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX11-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_ptr:
; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-GISEL-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT: .LBB10_1: ; %bb1
; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s5, v2
-; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s6, v3
-; GFX11-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX11-GISEL-TRUE16-NEXT: flat_load_b32 v2, v[2:3]
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX11-GISEL-TRUE16-NEXT: s_endpgm
;
-; GFX11-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX11-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_ptr:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-GISEL-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX11-GISEL-FAKE16-NEXT: .LBB10_1: ; %bb1
; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s5, v2
-; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s6, v3
-; GFX11-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX11-GISEL-FAKE16-NEXT: flat_load_b32 v2, v[2:3]
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX11-GISEL-FAKE16-NEXT: s_endpgm
;
-; GFX11-GISEL-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX11-GISEL-LABEL: struct_ptr_atomic_buffer_load_ptr:
; GFX11-GISEL: ; %bb.0: ; %bb
; GFX11-GISEL-NEXT: s_clause 0x1
; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: ; implicit-def: $sgpr5
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-GISEL-NEXT: .LBB8_1: ; %bb1
+; GFX11-GISEL-NEXT: .LBB10_1: ; %bb1
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-NEXT: v_readfirstlane_b32 s5, v2
-; GFX11-GISEL-NEXT: v_readfirstlane_b32 s6, v3
-; GFX11-GISEL-NEXT: s_pack_ll_b32_b16 s5, s5, s6
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX11-GISEL-NEXT: flat_load_b32 v2, v[2:3]
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX11-GISEL-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, s6
; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB8_1
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
; GFX11-GISEL-NEXT: s_endpgm
;
-; GFX12-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX12-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_ptr:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
; GFX12-SDAG-TRUE16-NEXT: global_wb
; GFX12-SDAG-TRUE16-NEXT: v_nop
@@ -675,20 +2527,21 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i16(ptr addrspace(8)
; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-SDAG-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: .LBB10_1: ; %bb1
; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT: flat_load_b32 v2, v[2:3]
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX12-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX12-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX12-SDAG-TRUE16-NEXT: s_endpgm
;
-; GFX12-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX12-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_ptr:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
; GFX12-SDAG-FAKE16-NEXT: global_wb
; GFX12-SDAG-FAKE16-NEXT: v_nop
@@ -701,22 +2554,21 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i16(ptr addrspace(8)
; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-SDAG-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: .LBB10_1: ; %bb1
; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX12-SDAG-FAKE16-NEXT: flat_load_b32 v2, v[2:3]
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX12-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX12-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX12-SDAG-FAKE16-NEXT: s_endpgm
;
-; GFX12-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX12-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_ptr:
; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
; GFX12-GISEL-TRUE16-NEXT: global_wb
; GFX12-GISEL-TRUE16-NEXT: v_nop
@@ -727,24 +2579,27 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i16(ptr addrspace(8)
; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT: ; implicit-def: $sgpr5
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-GISEL-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: .LBB10_1: ; %bb1
; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s5, v2
-; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s6, v3
-; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX12-GISEL-TRUE16-NEXT: flat_load_b32 v2, v[2:3]
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-TRUE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, s6
; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX12-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX12-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX12-GISEL-TRUE16-NEXT: s_endpgm
;
-; GFX12-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX12-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_ptr:
; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
; GFX12-GISEL-FAKE16-NEXT: global_wb
; GFX12-GISEL-FAKE16-NEXT: v_nop
@@ -755,143 +2610,25 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i16(ptr addrspace(8)
; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-GISEL-FAKE16-NEXT: ; implicit-def: $sgpr5
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-GISEL-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: .LBB10_1: ; %bb1
; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s5, v2
-; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s6, v3
-; GFX12-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX12-GISEL-FAKE16-NEXT: flat_load_b32 v2, v[2:3]
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 s5, s5, exec_lo
+; GFX12-GISEL-FAKE16-NEXT: s_and_b32 s6, exec_lo, s4
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, s6
; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
+; GFX12-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX12-GISEL-FAKE16-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- br label %bb1
-bb1:
- %load = call <4 x i16> @llvm.amdgcn.struct.ptr.atomic.buffer.load.v4i16(ptr addrspace(8) %ptr, i32 %index, i32 4, i32 0, i32 1)
- %shortened = shufflevector <4 x i16> %load, <4 x i16> poison, <2 x i32> <i32 0, i32 2>
- %bitcast = bitcast <2 x i16> %shortened to i32
- %cmp = icmp eq i32 %bitcast, %id
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-
-define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i32(ptr addrspace(8) %ptr, i32 %index) {
-; GFX11-LABEL: struct_ptr_atomic_buffer_load_v4i32:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: .LBB9_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], 0 idxen offset:4 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB9_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: struct_ptr_atomic_buffer_load_v4i32:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: .LBB9_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB9_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
-bb:
- %id = tail call i32 @llvm.amdgcn.workitem.id.x()
- br label %bb1
-bb1:
- %load = call <4 x i32> @llvm.amdgcn.struct.ptr.atomic.buffer.load.v4i32(ptr addrspace(8) %ptr, i32 %index, i32 4, i32 0, i32 1)
- %extracted = extractelement <4 x i32> %load, i32 3
- %cmp = icmp eq i32 %extracted, %id
- br i1 %cmp, label %bb1, label %bb2
-bb2:
- ret void
-}
-
-define amdgpu_kernel void @struct_ptr_atomic_buffer_load_ptr(ptr addrspace(8) %ptr, i32 %index) {
-; GFX11-LABEL: struct_ptr_atomic_buffer_load_ptr:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: .LBB10_1: ; %bb1
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: flat_load_b32 v2, v[2:3]
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB10_1
-; GFX11-NEXT: ; %bb.2: ; %bb2
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: struct_ptr_atomic_buffer_load_ptr:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: global_wb
-; GFX12-NEXT: v_nop
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_xcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: .LBB10_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: flat_load_b32 v2, v[2:3]
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB10_1
-; GFX12-NEXT: ; %bb.2: ; %bb2
-; GFX12-NEXT: s_endpgm
bb:
%id = tail call i32 @llvm.amdgcn.workitem.id.x()
br label %bb1
@@ -912,3 +2649,6 @@ declare <4 x i32> @llvm.amdgcn.struct.ptr.atom.buffer.load.v4i32(ptr addrspace(8
declare ptr @llvm.amdgcn.struct.ptr.atom.buffer.load.ptr(ptr addrspace(8), i32, i32, i32, i32 immarg)
declare i32 @llvm.amdgcn.struct.ptr.buffer.load.i32(ptr addrspace(8), i32, i32, i32, i32 immarg)
declare i32 @llvm.amdgcn.workitem.id.x()
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11: {{.*}}
+; GFX12: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/loop-on-function-argument.ll b/llvm/test/CodeGen/AMDGPU/loop-on-function-argument.ll
index 2445925b89bef..6b91f0e9ad247 100644
--- a/llvm/test/CodeGen/AMDGPU/loop-on-function-argument.ll
+++ b/llvm/test/CodeGen/AMDGPU/loop-on-function-argument.ll
@@ -7,13 +7,13 @@ define void @loop_on_argument(i1 %arg) {
; IR-NEXT: entry:
; IR-NEXT: br label [[LOOP:%.*]]
; IR: loop:
-; IR-NEXT: [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP0:%.*]], [[LOOP]] ], [ 0, [[ENTRY:%.*]] ]
-; IR-NEXT: [[TMP0]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[ARG:%.*]], i64 [[PHI_BROKEN]])
+; IR-NEXT: [[PHI_BROKEN:%.*]] = phi i1 [ [[TMP0:%.*]], [[LOOP]] ], [ false, [[ENTRY:%.*]] ]
+; IR-NEXT: [[TMP0]] = call i1 @llvm.amdgcn.if.break(i1 [[ARG:%.*]], i1 [[PHI_BROKEN]])
; IR-NEXT: store volatile i32 0, ptr addrspace(1) poison, align 4
-; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP0]])
+; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.loop(i1 [[TMP0]])
; IR-NEXT: br i1 [[TMP1]], label [[EXIT:%.*]], label [[LOOP]]
; IR: exit:
-; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP0]])
+; IR-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP0]])
; IR-NEXT: ret void
;
; CHECK-LABEL: loop_on_argument:
diff --git a/llvm/test/CodeGen/AMDGPU/loop_break.ll b/llvm/test/CodeGen/AMDGPU/loop_break.ll
index bcf9e77a15877..4a4201a6cc3c8 100644
--- a/llvm/test/CodeGen/AMDGPU/loop_break.ll
+++ b/llvm/test/CodeGen/AMDGPU/loop_break.ll
@@ -12,7 +12,7 @@ define amdgpu_kernel void @break_loop(i32 %arg) #0 {
; OPT-NEXT: [[MY_TMP:%.*]] = sub i32 [[ID]], [[ARG]]
; OPT-NEXT: br label %[[BB1:.*]]
; OPT: [[BB1]]:
-; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP2:%.*]], %[[FLOW:.*]] ], [ 0, %[[BB]] ]
+; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i1 [ [[TMP2:%.*]], %[[FLOW:.*]] ], [ false, %[[BB]] ]
; OPT-NEXT: [[LSR_IV:%.*]] = phi i32 [ undef, %[[BB]] ], [ [[TMP0:%.*]], %[[FLOW]] ]
; OPT-NEXT: [[TMP0]] = add i32 [[LSR_IV]], 1
; OPT-NEXT: [[CMP0:%.*]] = icmp slt i32 [[TMP0]], 0
@@ -23,11 +23,11 @@ define amdgpu_kernel void @break_loop(i32 %arg) #0 {
; OPT-NEXT: br label %[[FLOW]]
; OPT: [[FLOW]]:
; OPT-NEXT: [[TMP1:%.*]] = phi i1 [ [[CMP1]], %[[BB4]] ], [ true, %[[BB1]] ]
-; OPT-NEXT: [[TMP2]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[TMP1]], i64 [[PHI_BROKEN]])
-; OPT-NEXT: [[TMP3:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP2]])
+; OPT-NEXT: [[TMP2]] = call i1 @llvm.amdgcn.if.break(i1 [[TMP1]], i1 [[PHI_BROKEN]])
+; OPT-NEXT: [[TMP3:%.*]] = call i1 @llvm.amdgcn.loop(i1 [[TMP2]])
; OPT-NEXT: br i1 [[TMP3]], label %[[BB9:.*]], label %[[BB1]]
; OPT: [[BB9]]:
-; OPT-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP2]])
+; OPT-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP2]])
; OPT-NEXT: ret void
;
; GCN-LABEL: break_loop:
@@ -90,7 +90,7 @@ define amdgpu_kernel void @undef_phi_cond_break_loop(i32 %arg) #0 {
; OPT-NEXT: [[MY_TMP:%.*]] = sub i32 [[ID]], [[ARG]]
; OPT-NEXT: br label %[[BB1:.*]]
; OPT: [[BB1]]:
-; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP0:%.*]], %[[FLOW:.*]] ], [ 0, %[[BB]] ]
+; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i1 [ [[TMP0:%.*]], %[[FLOW:.*]] ], [ false, %[[BB]] ]
; OPT-NEXT: [[LSR_IV:%.*]] = phi i32 [ undef, %[[BB]] ], [ [[MY_TMP2:%.*]], %[[FLOW]] ]
; OPT-NEXT: [[LSR_IV_NEXT:%.*]] = add i32 [[LSR_IV]], 1
; OPT-NEXT: [[CMP0:%.*]] = icmp slt i32 [[LSR_IV_NEXT]], 0
@@ -102,11 +102,11 @@ define amdgpu_kernel void @undef_phi_cond_break_loop(i32 %arg) #0 {
; OPT: [[FLOW]]:
; OPT-NEXT: [[MY_TMP2]] = phi i32 [ [[LSR_IV_NEXT]], %[[BB4]] ], [ undef, %[[BB1]] ]
; OPT-NEXT: [[MY_TMP3:%.*]] = phi i1 [ [[CMP1]], %[[BB4]] ], [ undef, %[[BB1]] ]
-; OPT-NEXT: [[TMP0]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[MY_TMP3]], i64 [[PHI_BROKEN]])
-; OPT-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP0]])
+; OPT-NEXT: [[TMP0]] = call i1 @llvm.amdgcn.if.break(i1 [[MY_TMP3]], i1 [[PHI_BROKEN]])
+; OPT-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.loop(i1 [[TMP0]])
; OPT-NEXT: br i1 [[TMP1]], label %[[BB9:.*]], label %[[BB1]]
; OPT: [[BB9]]:
-; OPT-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP0]])
+; OPT-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP0]])
; OPT-NEXT: store volatile i32 7, ptr addrspace(3) poison, align 4
; OPT-NEXT: ret void
;
@@ -179,7 +179,7 @@ define amdgpu_kernel void @constexpr_phi_cond_break_loop(i32 %arg) #0 {
; OPT-NEXT: [[MY_TMP:%.*]] = sub i32 [[ID]], [[ARG]]
; OPT-NEXT: br label %[[BB1:.*]]
; OPT: [[BB1]]:
-; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP0:%.*]], %[[FLOW:.*]] ], [ 0, %[[BB]] ]
+; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i1 [ [[TMP0:%.*]], %[[FLOW:.*]] ], [ false, %[[BB]] ]
; OPT-NEXT: [[LSR_IV:%.*]] = phi i32 [ undef, %[[BB]] ], [ [[MY_TMP2:%.*]], %[[FLOW]] ]
; OPT-NEXT: [[LSR_IV_NEXT:%.*]] = add i32 [[LSR_IV]], 1
; OPT-NEXT: [[CMP0:%.*]] = icmp slt i32 [[LSR_IV_NEXT]], 0
@@ -192,11 +192,11 @@ define amdgpu_kernel void @constexpr_phi_cond_break_loop(i32 %arg) #0 {
; OPT: [[FLOW]]:
; OPT-NEXT: [[MY_TMP2]] = phi i32 [ [[LSR_IV_NEXT]], %[[BB4]] ], [ undef, %[[BB1]] ]
; OPT-NEXT: [[MY_TMP3:%.*]] = phi i1 [ [[CMP1]], %[[BB4]] ], [ [[CMP2]], %[[BB1]] ]
-; OPT-NEXT: [[TMP0]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[MY_TMP3]], i64 [[PHI_BROKEN]])
-; OPT-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP0]])
+; OPT-NEXT: [[TMP0]] = call i1 @llvm.amdgcn.if.break(i1 [[MY_TMP3]], i1 [[PHI_BROKEN]])
+; OPT-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.loop(i1 [[TMP0]])
; OPT-NEXT: br i1 [[TMP1]], label %[[BB9:.*]], label %[[BB1]]
; OPT: [[BB9]]:
-; OPT-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP0]])
+; OPT-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP0]])
; OPT-NEXT: store volatile i32 7, ptr addrspace(3) poison, align 4
; OPT-NEXT: ret void
;
@@ -271,7 +271,7 @@ define amdgpu_kernel void @true_phi_cond_break_loop(i32 %arg) #0 {
; OPT-NEXT: [[MY_TMP:%.*]] = sub i32 [[ID]], [[ARG]]
; OPT-NEXT: br label %[[BB1:.*]]
; OPT: [[BB1]]:
-; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP0:%.*]], %[[FLOW:.*]] ], [ 0, %[[BB]] ]
+; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i1 [ [[TMP0:%.*]], %[[FLOW:.*]] ], [ false, %[[BB]] ]
; OPT-NEXT: [[LSR_IV:%.*]] = phi i32 [ undef, %[[BB]] ], [ [[MY_TMP2:%.*]], %[[FLOW]] ]
; OPT-NEXT: [[LSR_IV_NEXT:%.*]] = add i32 [[LSR_IV]], 1
; OPT-NEXT: [[CMP0:%.*]] = icmp slt i32 [[LSR_IV_NEXT]], 0
@@ -283,11 +283,11 @@ define amdgpu_kernel void @true_phi_cond_break_loop(i32 %arg) #0 {
; OPT: [[FLOW]]:
; OPT-NEXT: [[MY_TMP2]] = phi i32 [ [[LSR_IV_NEXT]], %[[BB4]] ], [ undef, %[[BB1]] ]
; OPT-NEXT: [[MY_TMP3:%.*]] = phi i1 [ [[CMP1]], %[[BB4]] ], [ true, %[[BB1]] ]
-; OPT-NEXT: [[TMP0]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[MY_TMP3]], i64 [[PHI_BROKEN]])
-; OPT-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP0]])
+; OPT-NEXT: [[TMP0]] = call i1 @llvm.amdgcn.if.break(i1 [[MY_TMP3]], i1 [[PHI_BROKEN]])
+; OPT-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.loop(i1 [[TMP0]])
; OPT-NEXT: br i1 [[TMP1]], label %[[BB9:.*]], label %[[BB1]]
; OPT: [[BB9]]:
-; OPT-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP0]])
+; OPT-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP0]])
; OPT-NEXT: store volatile i32 7, ptr addrspace(3) poison, align 4
; OPT-NEXT: ret void
;
@@ -361,7 +361,7 @@ define amdgpu_kernel void @false_phi_cond_break_loop(i32 %arg) #0 {
; OPT-NEXT: [[MY_TMP:%.*]] = sub i32 [[ID]], [[ARG]]
; OPT-NEXT: br label %[[BB1:.*]]
; OPT: [[BB1]]:
-; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP0:%.*]], %[[FLOW:.*]] ], [ 0, %[[BB]] ]
+; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i1 [ [[TMP0:%.*]], %[[FLOW:.*]] ], [ false, %[[BB]] ]
; OPT-NEXT: [[LSR_IV:%.*]] = phi i32 [ undef, %[[BB]] ], [ [[MY_TMP2:%.*]], %[[FLOW]] ]
; OPT-NEXT: [[LSR_IV_NEXT:%.*]] = add i32 [[LSR_IV]], 1
; OPT-NEXT: [[CMP0:%.*]] = icmp slt i32 [[LSR_IV_NEXT]], 0
@@ -373,11 +373,11 @@ define amdgpu_kernel void @false_phi_cond_break_loop(i32 %arg) #0 {
; OPT: [[FLOW]]:
; OPT-NEXT: [[MY_TMP2]] = phi i32 [ [[LSR_IV_NEXT]], %[[BB4]] ], [ undef, %[[BB1]] ]
; OPT-NEXT: [[MY_TMP3:%.*]] = phi i1 [ [[CMP1]], %[[BB4]] ], [ false, %[[BB1]] ]
-; OPT-NEXT: [[TMP0]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[MY_TMP3]], i64 [[PHI_BROKEN]])
-; OPT-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP0]])
+; OPT-NEXT: [[TMP0]] = call i1 @llvm.amdgcn.if.break(i1 [[MY_TMP3]], i1 [[PHI_BROKEN]])
+; OPT-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.loop(i1 [[TMP0]])
; OPT-NEXT: br i1 [[TMP1]], label %[[BB9:.*]], label %[[BB1]]
; OPT: [[BB9]]:
-; OPT-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP0]])
+; OPT-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP0]])
; OPT-NEXT: store volatile i32 7, ptr addrspace(3) poison, align 4
; OPT-NEXT: ret void
;
@@ -454,7 +454,7 @@ define amdgpu_kernel void @invert_true_phi_cond_break_loop(i32 %arg) #0 {
; OPT-NEXT: [[MY_TMP:%.*]] = sub i32 [[ID]], [[ARG]]
; OPT-NEXT: br label %[[BB1:.*]]
; OPT: [[BB1]]:
-; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP0:%.*]], %[[FLOW:.*]] ], [ 0, %[[BB]] ]
+; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i1 [ [[TMP0:%.*]], %[[FLOW:.*]] ], [ false, %[[BB]] ]
; OPT-NEXT: [[LSR_IV:%.*]] = phi i32 [ undef, %[[BB]] ], [ [[MY_TMP2:%.*]], %[[FLOW]] ]
; OPT-NEXT: [[LSR_IV_NEXT:%.*]] = add i32 [[LSR_IV]], 1
; OPT-NEXT: [[CMP0:%.*]] = icmp slt i32 [[LSR_IV_NEXT]], 0
@@ -467,11 +467,11 @@ define amdgpu_kernel void @invert_true_phi_cond_break_loop(i32 %arg) #0 {
; OPT-NEXT: [[MY_TMP2]] = phi i32 [ [[LSR_IV_NEXT]], %[[BB4]] ], [ undef, %[[BB1]] ]
; OPT-NEXT: [[MY_TMP3:%.*]] = phi i1 [ [[CMP1]], %[[BB4]] ], [ true, %[[BB1]] ]
; OPT-NEXT: [[MY_TMP3_INV:%.*]] = xor i1 [[MY_TMP3]], true
-; OPT-NEXT: [[TMP0]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[MY_TMP3_INV]], i64 [[PHI_BROKEN]])
-; OPT-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP0]])
+; OPT-NEXT: [[TMP0]] = call i1 @llvm.amdgcn.if.break(i1 [[MY_TMP3_INV]], i1 [[PHI_BROKEN]])
+; OPT-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.loop(i1 [[TMP0]])
; OPT-NEXT: br i1 [[TMP1]], label %[[BB9:.*]], label %[[BB1]]
; OPT: [[BB9]]:
-; OPT-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP0]])
+; OPT-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP0]])
; OPT-NEXT: store volatile i32 7, ptr addrspace(3) poison, align 4
; OPT-NEXT: ret void
;
diff --git a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll
index bca93a1d91ef1..03926a58767b7 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll
+++ b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll
@@ -121,8 +121,8 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
; CHECK-NEXT: s_cbranch_execz .LBB0_25
; CHECK-NEXT: ; %bb.1: ; %.preheader5
; CHECK-NEXT: v_mul_lo_u32 v44, v41, 14
-; CHECK-NEXT: s_mov_b32 s4, 0
; CHECK-NEXT: s_mov_b32 s5, 0
+; CHECK-NEXT: s_mov_b32 s4, 0
; CHECK-NEXT: v_add_nc_u32_e32 v45, 0x3c04, v44
; CHECK-NEXT: .LBB0_2: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: v_add_nc_u32_e32 v0, s5, v45
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-atomic-insert-end.mir b/llvm/test/CodeGen/AMDGPU/memory-legalizer-atomic-insert-end.mir
index 5afaa7065f87f..72f83fedaa2de 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-atomic-insert-end.mir
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-atomic-insert-end.mir
@@ -15,9 +15,9 @@
%ptr = load volatile ptr addrspace(1), ptr addrspace(1) %tid.gep
%xor = xor i32 %tid, 1
%cmp = icmp ne i32 %xor, 0
- %1 = call { i1, i64 } @llvm.amdgcn.if(i1 %cmp)
- %2 = extractvalue { i1, i64 } %1, 0
- %3 = extractvalue { i1, i64 } %1, 1
+ %1 = call { i1, i1 } @llvm.amdgcn.if(i1 %cmp)
+ %2 = extractvalue { i1, i1 } %1, 0
+ %3 = extractvalue { i1, i1 } %1, 1
br i1 %2, label %atomic, label %exit
atomic: ; preds = %0
@@ -26,13 +26,13 @@
br label %exit
exit: ; preds = %atomic, %0
- call void @llvm.amdgcn.end.cf(i64 %3)
+ call void @llvm.amdgcn.end.cf(i1 %3)
ret void
}
- declare { i1, i64 } @llvm.amdgcn.if(i1)
+ declare { i1, i1 } @llvm.amdgcn.if(i1)
- declare void @llvm.amdgcn.end.cf(i64)
+ declare void @llvm.amdgcn.end.cf(i1)
attributes #0 = { nounwind readnone }
attributes #1 = { nounwind "target-cpu"="gfx803" }
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-nontemporal-1.mir b/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-nontemporal-1.mir
index 185643dffded2..8c46a0d37de8b 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-nontemporal-1.mir
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-nontemporal-1.mir
@@ -29,22 +29,22 @@
}
; Function Attrs: convergent nounwind
- declare { i1, i64 } @llvm.amdgcn.if(i1) #1
+ declare { i1, i1 } @llvm.amdgcn.if(i1) #1
; Function Attrs: convergent nounwind
- declare { i1, i64 } @llvm.amdgcn.else(i64) #1
+ declare { i1, i1 } @llvm.amdgcn.else(i1) #1
; Function Attrs: convergent nounwind readnone
declare i64 @llvm.amdgcn.break(i64) #2
; Function Attrs: convergent nounwind readnone
- declare i64 @llvm.amdgcn.if.break(i1, i64) #2
+ declare i1 @llvm.amdgcn.if.break(i1, i1) #2
; Function Attrs: convergent nounwind
- declare i1 @llvm.amdgcn.loop(i64) #1
+ declare i1 @llvm.amdgcn.loop(i1) #1
; Function Attrs: convergent nounwind
- declare void @llvm.amdgcn.end.cf(i64) #1
+ declare void @llvm.amdgcn.end.cf(i1) #1
attributes #0 = { "target-cpu"="gfx803" }
attributes #1 = { convergent nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
index 2122445cafa7e..d99b47ea8b32a 100644
--- a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
+++ b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
@@ -26,13 +26,13 @@ define void @memset_p0_varsize_align_4_varsetval(ptr addrspace(0) align 4 %dst,
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: .LBB0_2: ; %dynamic-memset-expansion-main-body
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[14:15], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 16
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[12:13]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: v_lshl_add_u64 v[14:15], v[0:1], 0, s[0:1]
+; GFX942-NEXT: s_add_u32 s0, s0, 16
+; GFX942-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[12:13]
+; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: flat_store_dwordx4 v[14:15], v[4:7]
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_cbranch_execnz .LBB0_2
; GFX942-NEXT: .LBB0_3: ; %Flow4
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
@@ -46,13 +46,13 @@ define void @memset_p0_varsize_align_4_varsetval(ptr addrspace(0) align 4 %dst,
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: .LBB0_5: ; %dynamic-memset-expansion-residual-body
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
+; GFX942-NEXT: s_add_u32 s0, s0, 1
+; GFX942-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[8:9]
+; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: flat_store_byte v[4:5], v2
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_cbranch_execnz .LBB0_5
; GFX942-NEXT: .LBB0_6: ; %Flow2
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
@@ -86,13 +86,13 @@ define void @memset_p1_varsize_align_4_varsetval(ptr addrspace(1) align 4 %dst,
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: .LBB1_2: ; %dynamic-memset-expansion-main-body
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[14:15], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 16
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[12:13]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: v_lshl_add_u64 v[14:15], v[0:1], 0, s[0:1]
+; GFX942-NEXT: s_add_u32 s0, s0, 16
+; GFX942-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[12:13]
+; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: global_store_dwordx4 v[14:15], v[4:7], off
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_cbranch_execnz .LBB1_2
; GFX942-NEXT: .LBB1_3: ; %Flow4
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
@@ -106,13 +106,13 @@ define void @memset_p1_varsize_align_4_varsetval(ptr addrspace(1) align 4 %dst,
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: .LBB1_5: ; %dynamic-memset-expansion-residual-body
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
+; GFX942-NEXT: s_add_u32 s0, s0, 1
+; GFX942-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[8:9]
+; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: global_store_byte v[4:5], v2, off
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_cbranch_execnz .LBB1_5
; GFX942-NEXT: .LBB1_6: ; %Flow2
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
@@ -141,18 +141,18 @@ define void @memset_p3_varsize_align_4_varsetval(ptr addrspace(3) align 4 %dst,
; GFX942-NEXT: v_mov_b32_e32 v3, v6
; GFX942-NEXT: v_mov_b32_e32 v8, v6
; GFX942-NEXT: v_mov_b32_e32 v7, v6
-; GFX942-NEXT: v_mov_b32_e32 v9, v0
; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_mov_b32_e32 v9, v0
; GFX942-NEXT: .LBB2_2: ; %dynamic-memset-expansion-main-body
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s4, s4, 16
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[4:5]
+; GFX942-NEXT: s_add_u32 s0, s0, 16
+; GFX942-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[4:5]
; GFX942-NEXT: ds_write2_b32 v9, v8, v7 offset0:2 offset1:3
; GFX942-NEXT: ds_write2_b32 v9, v6, v3 offset1:1
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: v_add_u32_e32 v9, 16, v9
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_cbranch_execnz .LBB2_2
; GFX942-NEXT: .LBB2_3: ; %Flow7
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
@@ -166,13 +166,13 @@ define void @memset_p3_varsize_align_4_varsetval(ptr addrspace(3) align 4 %dst,
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: .LBB2_5: ; %dynamic-memset-expansion-residual-body
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
+; GFX942-NEXT: s_add_u32 s0, s0, 1
+; GFX942-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[10:11]
; GFX942-NEXT: ds_write_b8 v0, v1
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: v_add_u32_e32 v0, 1, v0
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_cbranch_execnz .LBB2_5
; GFX942-NEXT: .LBB2_6: ; %Flow5
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
@@ -201,17 +201,17 @@ define void @memset_p5_varsize_align_4_varsetval(ptr addrspace(5) align 4 %dst,
; GFX942-NEXT: v_mov_b32_e32 v7, v6
; GFX942-NEXT: v_mov_b32_e32 v8, v6
; GFX942-NEXT: v_mov_b32_e32 v9, v6
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
; GFX942-NEXT: .LBB3_2: ; %dynamic-memset-expansion-main-body
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s4, s4, 16
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[4:5]
+; GFX942-NEXT: s_add_u32 s0, s0, 16
+; GFX942-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[4:5]
; GFX942-NEXT: scratch_store_dwordx4 v3, v[6:9], off
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: v_add_u32_e32 v3, 16, v3
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_cbranch_execnz .LBB3_2
; GFX942-NEXT: .LBB3_3: ; %Flow7
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
@@ -225,13 +225,13 @@ define void @memset_p5_varsize_align_4_varsetval(ptr addrspace(5) align 4 %dst,
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: .LBB3_5: ; %dynamic-memset-expansion-residual-body
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
+; GFX942-NEXT: s_add_u32 s0, s0, 1
+; GFX942-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[10:11]
; GFX942-NEXT: scratch_store_byte v0, v1, off
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-NEXT: v_add_u32_e32 v0, 1, v0
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_cbranch_execnz .LBB3_5
; GFX942-NEXT: .LBB3_6: ; %Flow5
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
@@ -982,116 +982,272 @@ entry:
}
define void @memset_p1_varsz_align_4_set40(ptr addrspace(1) align 4 %dst, i64 %size) #0 {
-; GFX942-LABEL: memset_p1_varsz_align_4_set40:
-; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_and_b32_e32 v10, -16, v2
-; GFX942-NEXT: v_mov_b32_e32 v11, v3
-; GFX942-NEXT: v_and_b32_e32 v8, 15, v2
-; GFX942-NEXT: v_mov_b32_e32 v9, 0
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB12_3
-; GFX942-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
-; GFX942-NEXT: v_mov_b32_e32 v4, 0x28282828
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-NEXT: v_mov_b32_e32 v6, v4
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB12_2: ; %dynamic-memset-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 16
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB12_2
-; GFX942-NEXT: .LBB12_3: ; %Flow4
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB12_6
-; GFX942-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-NEXT: v_and_b32_e32 v2, -16, v2
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v2, 40
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB12_5: ; %dynamic-memset-expansion-residual-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: global_store_byte v[4:5], v2, off
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB12_5
-; GFX942-NEXT: .LBB12_6: ; %Flow2
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_p1_varsz_align_4_set40:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v3
+; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB12_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, 0x28282828
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-SDAG-NEXT: .LBB12_2: ; %dynamic-memset-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 16
+; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[10:11]
+; GFX942-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB12_2
+; GFX942-SDAG-NEXT: .LBB12_3: ; %Flow4
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB12_6
+; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 40
+; GFX942-SDAG-NEXT: .LBB12_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[8:9]
+; GFX942-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-SDAG-NEXT: global_store_byte v[4:5], v2, off
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB12_5
+; GFX942-SDAG-NEXT: .LBB12_6: ; %Flow2
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_p1_varsz_align_4_set40:
+; GFX942-GISEL: ; %bb.0: ; %entry
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], 15
+; GFX942-GISEL-NEXT: v_and_b32_e32 v8, 15, v2
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v10, vcc, v2, v8
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v11, vcc, 0, v3, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB12_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: s_mov_b32 s8, 0x28282828
+; GFX942-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[10:11]
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
+; GFX942-GISEL-NEXT: .LBB12_2: ; %dynamic-memset-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[0:1]
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 16
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v12, vcc, v0, v12
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v13, vcc, v1, v13, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[10:11]
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX942-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB12_2
+; GFX942-GISEL-NEXT: .LBB12_3: ; %Flow4
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB12_6
+; GFX942-GISEL-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-GISEL-NEXT: v_lshrrev_b64 v[2:3], 4, v[2:3]
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 4, v[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 40
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
+; GFX942-GISEL-NEXT: .LBB12_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v4, vcc, v0, v4
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v5, vcc, v1, v5, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX942-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
+; GFX942-GISEL-NEXT: global_store_byte v[4:5], v2, off
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB12_5
+; GFX942-GISEL-NEXT: .LBB12_6: ; %Flow2
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
tail call void @llvm.memset.p1.i64(ptr addrspace(1) noundef nonnull align 4 %dst, i8 40, i64 %size, i1 false)
ret void
}
define void @memset_p1_varsz_align_4_set0(ptr addrspace(1) align 4 %dst, i64 %size) #0 {
-; GFX942-LABEL: memset_p1_varsz_align_4_set0:
-; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_and_b32_e32 v6, -16, v2
-; GFX942-NEXT: v_mov_b32_e32 v7, v3
-; GFX942-NEXT: v_and_b32_e32 v4, 15, v2
-; GFX942-NEXT: v_mov_b32_e32 v5, 0
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB13_3
-; GFX942-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
-; GFX942-NEXT: v_mov_b32_e32 v8, v5
-; GFX942-NEXT: v_mov_b32_e32 v9, v5
-; GFX942-NEXT: v_mov_b32_e32 v10, v5
-; GFX942-NEXT: v_mov_b32_e32 v11, v5
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB13_2: ; %dynamic-memset-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 16
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: global_store_dwordx4 v[12:13], v[8:11], off
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB13_2
-; GFX942-NEXT: .LBB13_3: ; %Flow4
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB13_6
-; GFX942-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-NEXT: v_and_b32_e32 v2, -16, v2
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v2, 0
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB13_5: ; %dynamic-memset-expansion-residual-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[4:5]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: global_store_byte v[6:7], v2, off
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB13_5
-; GFX942-NEXT: .LBB13_6: ; %Flow2
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_p1_varsz_align_4_set0:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_and_b32_e32 v6, -16, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v3
+; GFX942-SDAG-NEXT: v_and_b32_e32 v4, 15, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB13_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v5
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v5
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v5
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v5
+; GFX942-SDAG-NEXT: .LBB13_2: ; %dynamic-memset-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 16
+; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[6:7]
+; GFX942-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[12:13], v[8:11], off
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB13_2
+; GFX942-SDAG-NEXT: .LBB13_3: ; %Flow4
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB13_6
+; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-SDAG-NEXT: .LBB13_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[4:5]
+; GFX942-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-SDAG-NEXT: global_store_byte v[6:7], v2, off
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB13_5
+; GFX942-SDAG-NEXT: .LBB13_6: ; %Flow2
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_p1_varsz_align_4_set0:
+; GFX942-GISEL: ; %bb.0: ; %entry
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], 15
+; GFX942-GISEL-NEXT: v_and_b32_e32 v8, 15, v2
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v10, vcc, v2, v8
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v11, vcc, 0, v3, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB13_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: s_mov_b32 s8, 0
+; GFX942-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[10:11]
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
+; GFX942-GISEL-NEXT: .LBB13_2: ; %dynamic-memset-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[0:1]
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 16
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v12, vcc, v0, v12
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v13, vcc, v1, v13, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[10:11]
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX942-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB13_2
+; GFX942-GISEL-NEXT: .LBB13_3: ; %Flow4
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB13_6
+; GFX942-GISEL-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-GISEL-NEXT: v_lshrrev_b64 v[2:3], 4, v[2:3]
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 4, v[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
+; GFX942-GISEL-NEXT: .LBB13_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v4, vcc, v0, v4
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v5, vcc, v1, v5, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX942-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
+; GFX942-GISEL-NEXT: global_store_byte v[4:5], v2, off
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB13_5
+; GFX942-GISEL-NEXT: .LBB13_6: ; %Flow2
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
tail call void @llvm.memset.p1.i64(ptr addrspace(1) noundef nonnull align 4 %dst, i8 0, i64 %size, i1 false)
ret void
@@ -1104,6 +1260,3 @@ declare void @llvm.memset.p5.i64(ptr addrspace(5) noalias nocapture writeonly, i
attributes #0 = { nounwind }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX942-GISEL: {{.*}}
-; GFX942-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/memset-pattern.ll b/llvm/test/CodeGen/AMDGPU/memset-pattern.ll
index 26a866c25c10d..812c47e2b03a7 100644
--- a/llvm/test/CodeGen/AMDGPU/memset-pattern.ll
+++ b/llvm/test/CodeGen/AMDGPU/memset-pattern.ll
@@ -317,185 +317,417 @@ define void @memset_pattern_i128_len16(ptr addrspace(1) align 16 %a) {
}
define void @memset_pattern_i128_dynlen(ptr addrspace(1) align 16 %a, i64 %len) {
-; GFX942-LABEL: memset_pattern_i128_dynlen:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB7_3
-; GFX942-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
-; GFX942-NEXT: v_mov_b32_e32 v4, 0xdddddddd
-; GFX942-NEXT: v_mov_b32_e32 v5, 0xcccccccc
-; GFX942-NEXT: v_mov_b32_e32 v6, 0xbbbbbbbb
-; GFX942-NEXT: v_mov_b32_e32 v7, 0xaaaaaaaa
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB7_2: ; %memset.pattern-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
-; GFX942-NEXT: global_store_dwordx4 v[0:1], v[4:7], off
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, 16
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB7_2
-; GFX942-NEXT: .LBB7_3: ; %Flow1
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_pattern_i128_dynlen:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB7_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, 0xdddddddd
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, 0xcccccccc
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, 0xbbbbbbbb
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, 0xaaaaaaaa
+; GFX942-SDAG-NEXT: .LBB7_2: ; %memset.pattern-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[2:3]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[0:1], v[4:7], off
+; GFX942-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, 16
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB7_2
+; GFX942-SDAG-NEXT: .LBB7_3: ; %Flow1
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_pattern_i128_dynlen:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB7_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: s_mov_b32 s8, 0xdddddddd
+; GFX942-GISEL-NEXT: s_mov_b32 s9, 0xcccccccc
+; GFX942-GISEL-NEXT: s_mov_b32 s10, 0xbbbbbbbb
+; GFX942-GISEL-NEXT: s_mov_b32 s11, 0xaaaaaaaa
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[10:11]
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
+; GFX942-GISEL-NEXT: .LBB7_2: ; %memset.pattern-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[0:1], v[4:7], off
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 16, v0
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[2:3]
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX942-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB7_2
+; GFX942-GISEL-NEXT: .LBB7_3: ; %Flow1
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
call void @llvm.experimental.memset.pattern(ptr addrspace(1) %a, i128 u0xaaaaaaaabbbbbbbbccccccccdddddddd, i64 %len, i1 false)
ret void
}
define void @memset_pattern_i128_dynlen_lds(ptr addrspace(3) align 16 %a, i64 %len) {
-; GFX942-LABEL: memset_pattern_i128_dynlen_lds:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-NEXT: v_mov_b32_e32 v6, v1
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB8_3
-; GFX942-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
-; GFX942-NEXT: v_mov_b32_e32 v2, 0xdddddddd
-; GFX942-NEXT: v_mov_b32_e32 v3, 0xcccccccc
-; GFX942-NEXT: v_mov_b32_e32 v4, 0xbbbbbbbb
-; GFX942-NEXT: v_mov_b32_e32 v5, 0xaaaaaaaa
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB8_2: ; %memset.pattern-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
-; GFX942-NEXT: ds_write_b128 v0, v[2:5]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_add_u32_e32 v0, 16, v0
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB8_2
-; GFX942-NEXT: .LBB8_3: ; %Flow1
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_pattern_i128_dynlen_lds:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v1
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB8_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 0xdddddddd
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 0xcccccccc
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, 0xbbbbbbbb
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, 0xaaaaaaaa
+; GFX942-SDAG-NEXT: .LBB8_2: ; %memset.pattern-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[6:7]
+; GFX942-SDAG-NEXT: ds_write_b128 v0, v[2:5]
+; GFX942-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-SDAG-NEXT: v_add_u32_e32 v0, 16, v0
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB8_2
+; GFX942-SDAG-NEXT: .LBB8_3: ; %Flow1
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_pattern_i128_dynlen_lds:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, v1
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v2
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB8_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: s_mov_b32 s8, 0xdddddddd
+; GFX942-GISEL-NEXT: s_mov_b32 s9, 0xcccccccc
+; GFX942-GISEL-NEXT: s_mov_b32 s10, 0xbbbbbbbb
+; GFX942-GISEL-NEXT: s_mov_b32 s11, 0xaaaaaaaa
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11]
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
+; GFX942-GISEL-NEXT: .LBB8_2: ; %memset.pattern-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 1
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[6:7]
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX942-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX942-GISEL-NEXT: ds_write_b128 v0, v[2:5]
+; GFX942-GISEL-NEXT: v_add_u32_e32 v0, 16, v0
+; GFX942-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB8_2
+; GFX942-GISEL-NEXT: .LBB8_3: ; %Flow1
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
call void @llvm.experimental.memset.pattern(ptr addrspace(3) align 16 %a, i128 u0xaaaaaaaabbbbbbbbccccccccdddddddd, i64 %len, i1 false)
ret void
}
define void @memset_pattern_i32_dynlen(ptr addrspace(1) align 16 %a, i64 %len) {
-; GFX942-LABEL: memset_pattern_i32_dynlen:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_and_b32_e32 v10, -4, v2
-; GFX942-NEXT: v_mov_b32_e32 v11, v3
-; GFX942-NEXT: v_and_b32_e32 v8, 3, v2
-; GFX942-NEXT: v_mov_b32_e32 v9, 0
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB9_3
-; GFX942-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
-; GFX942-NEXT: v_mov_b32_e32 v4, 0xaabbccdd
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-NEXT: v_mov_b32_e32 v6, v4
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: v_mov_b64_e32 v[12:13], v[0:1]
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB9_2: ; %memset.pattern-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s4, s4, 4
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
-; GFX942-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_lshl_add_u64 v[12:13], v[12:13], 0, 64
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB9_2
-; GFX942-NEXT: .LBB9_3: ; %Flow6
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB9_6
-; GFX942-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
-; GFX942-NEXT: v_lshlrev_b64 v[2:3], 2, v[2:3]
-; GFX942-NEXT: v_and_b32_e32 v2, -16, v2
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v2, 0xaabbccdd
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB9_5: ; %memset.pattern-expansion-residual-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
-; GFX942-NEXT: global_store_dword v[0:1], v2, off
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, 4
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB9_5
-; GFX942-NEXT: .LBB9_6: ; %Flow4
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_pattern_i32_dynlen:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -4, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v3
+; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 3, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB9_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, 0xaabbccdd
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[12:13], v[0:1]
+; GFX942-SDAG-NEXT: .LBB9_2: ; %memset.pattern-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 4
+; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[10:11]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
+; GFX942-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[12:13], 0, 64
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB9_2
+; GFX942-SDAG-NEXT: .LBB9_3: ; %Flow6
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB9_6
+; GFX942-SDAG-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
+; GFX942-SDAG-NEXT: v_lshlrev_b64 v[2:3], 2, v[2:3]
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 0xaabbccdd
+; GFX942-SDAG-NEXT: .LBB9_5: ; %memset.pattern-expansion-residual-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[8:9]
+; GFX942-SDAG-NEXT: global_store_dword v[0:1], v2, off
+; GFX942-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, 4
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB9_5
+; GFX942-SDAG-NEXT: .LBB9_6: ; %Flow4
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_pattern_i32_dynlen:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], 3
+; GFX942-GISEL-NEXT: v_and_b32_e32 v8, 3, v2
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v10, vcc, v2, v8
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v11, vcc, 0, v3, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB9_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: s_mov_b32 s8, 0xaabbccdd
+; GFX942-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], v[0:1]
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
+; GFX942-GISEL-NEXT: .LBB9_2: ; %memset.pattern-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 4
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v12, vcc, 64, v12
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[10:11]
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX942-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB9_2
+; GFX942-GISEL-NEXT: .LBB9_3: ; %Flow6
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB9_6
+; GFX942-GISEL-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
+; GFX942-GISEL-NEXT: v_lshrrev_b64 v[2:3], 2, v[2:3]
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 4, v[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 0xaabbccdd
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
+; GFX942-GISEL-NEXT: .LBB9_5: ; %memset.pattern-expansion-residual-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: global_store_dword v[0:1], v2, off
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 4, v0
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX942-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB9_5
+; GFX942-GISEL-NEXT: .LBB9_6: ; %Flow4
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
call void @llvm.experimental.memset.pattern(ptr addrspace(1) align 16 %a, i32 u0xaabbccdd, i64 %len, i1 false)
ret void
}
define void @memset_pattern_i32_dynval_dynlen(ptr addrspace(1) align 16 %a, i32 %val, i64 %len) {
-; GFX942-LABEL: memset_pattern_i32_dynval_dynlen:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v8, v3
-; GFX942-NEXT: v_and_b32_e32 v10, -4, v8
-; GFX942-NEXT: v_mov_b32_e32 v11, v4
-; GFX942-NEXT: v_mov_b32_e32 v9, v4
-; GFX942-NEXT: v_and_b32_e32 v6, 3, v8
-; GFX942-NEXT: v_mov_b32_e32 v7, 0
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB10_3
-; GFX942-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_mov_b32_e32 v4, v2
-; GFX942-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-NEXT: v_mov_b64_e32 v[12:13], v[0:1]
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB10_2: ; %memset.pattern-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s4, s4, 4
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
-; GFX942-NEXT: global_store_dwordx4 v[12:13], v[2:5], off
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_lshl_add_u64 v[12:13], v[12:13], 0, 64
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB10_2
-; GFX942-NEXT: .LBB10_3: ; %Flow6
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB10_6
-; GFX942-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
-; GFX942-NEXT: v_lshlrev_b64 v[4:5], 2, v[8:9]
-; GFX942-NEXT: v_and_b32_e32 v4, -16, v4
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[4:5]
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB10_5: ; %memset.pattern-expansion-residual-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
-; GFX942-NEXT: global_store_dword v[0:1], v2, off
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, 4
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB10_5
-; GFX942-NEXT: .LBB10_6: ; %Flow4
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_pattern_i32_dynval_dynlen:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v3
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -4, v8
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v4
+; GFX942-SDAG-NEXT: v_and_b32_e32 v6, 3, v8
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB10_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[12:13], v[0:1]
+; GFX942-SDAG-NEXT: .LBB10_2: ; %memset.pattern-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 4
+; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[10:11]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[12:13], v[2:5], off
+; GFX942-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[12:13], 0, 64
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB10_2
+; GFX942-SDAG-NEXT: .LBB10_3: ; %Flow6
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB10_6
+; GFX942-SDAG-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
+; GFX942-SDAG-NEXT: v_lshlrev_b64 v[4:5], 2, v[8:9]
+; GFX942-SDAG-NEXT: v_and_b32_e32 v4, -16, v4
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[4:5]
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB10_5: ; %memset.pattern-expansion-residual-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[6:7]
+; GFX942-SDAG-NEXT: global_store_dword v[0:1], v2, off
+; GFX942-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, 4
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB10_5
+; GFX942-SDAG-NEXT: .LBB10_6: ; %Flow4
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_pattern_i32_dynval_dynlen:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, v3
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], 3
+; GFX942-GISEL-NEXT: v_and_b32_e32 v6, 3, v8
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, v4
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v10, vcc, v8, v6
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v11, vcc, 0, v9, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB10_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], v[0:1]
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
+; GFX942-GISEL-NEXT: .LBB10_2: ; %memset.pattern-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[12:13], v[2:5], off
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 4
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v12, vcc, 64, v12
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[10:11]
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX942-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB10_2
+; GFX942-GISEL-NEXT: .LBB10_3: ; %Flow6
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB10_6
+; GFX942-GISEL-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
+; GFX942-GISEL-NEXT: v_lshrrev_b64 v[4:5], 2, v[8:9]
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[4:5], 4, v[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
+; GFX942-GISEL-NEXT: .LBB10_5: ; %memset.pattern-expansion-residual-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: global_store_dword v[0:1], v2, off
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 4, v0
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[6:7]
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX942-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB10_5
+; GFX942-GISEL-NEXT: .LBB10_6: ; %Flow4
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
call void @llvm.experimental.memset.pattern(ptr addrspace(1) align 16 %a, i32 %val, i64 %len, i1 false)
ret void
}
@@ -503,152 +735,336 @@ define void @memset_pattern_i32_dynval_dynlen(ptr addrspace(1) align 16 %a, i32
; For i96, the store size and the alloc size differ on amdgpu, this case is not
; optimized.
define void @memset_pattern_i96_dynval_dynlen(ptr addrspace(1) align 16 %a, i96 %val, i64 %len) {
-; GFX942-LABEL: memset_pattern_i96_dynval_dynlen:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v7, v6
-; GFX942-NEXT: v_mov_b32_e32 v6, v5
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB11_3
-; GFX942-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB11_2: ; %memset.pattern-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
-; GFX942-NEXT: global_store_dwordx3 v[0:1], v[2:4], off
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, 16
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB11_2
-; GFX942-NEXT: .LBB11_3: ; %Flow1
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_pattern_i96_dynval_dynlen:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v6
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v5
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB11_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB11_2: ; %memset.pattern-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[6:7]
+; GFX942-SDAG-NEXT: global_store_dwordx3 v[0:1], v[2:4], off
+; GFX942-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, 16
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB11_2
+; GFX942-SDAG-NEXT: .LBB11_3: ; %Flow1
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_pattern_i96_dynval_dynlen:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, v5
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, v6
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB11_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr8_sgpr9
+; GFX942-GISEL-NEXT: .LBB11_2: ; %memset.pattern-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: global_store_dwordx3 v[0:1], v[2:4], off
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 16, v0
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[10:11], exec, s[4:5]
+; GFX942-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[10:11], exec, s[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB11_2
+; GFX942-GISEL-NEXT: .LBB11_3: ; %Flow1
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
call void @llvm.experimental.memset.pattern(ptr addrspace(1) align 16 %a, i96 %val, i64 %len, i1 false)
ret void
}
define void @memset_pattern_i64_as7_dynlen(ptr addrspace(7) inreg align 16 %a, i32 %offset, i64 %len) {
-; GFX942-LABEL: memset_pattern_i64_as7_dynlen:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v8, v1
-; GFX942-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-NEXT: v_and_b32_e32 v10, -2, v8
-; GFX942-NEXT: v_and_b32_e32 v6, 1, v8
-; GFX942-NEXT: v_mov_b32_e32 v7, 0
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
-; GFX942-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB12_3
-; GFX942-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
-; GFX942-NEXT: v_mov_b32_e32 v2, 0xccccdddd
-; GFX942-NEXT: v_mov_b32_e32 v3, 0xaaaabbbb
-; GFX942-NEXT: v_add_u32_e32 v1, s16, v0
-; GFX942-NEXT: v_mov_b32_e32 v4, v2
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
-; GFX942-NEXT: s_mov_b64 s[8:9], 0
-; GFX942-NEXT: .LBB12_2: ; %memset.pattern-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s8, s8, 2
-; GFX942-NEXT: s_addc_u32 s9, s9, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[10:11]
-; GFX942-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen
-; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-NEXT: v_add_u32_e32 v1, 32, v1
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX942-NEXT: s_cbranch_execnz .LBB12_2
-; GFX942-NEXT: .LBB12_3: ; %Flow3
-; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GFX942-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB12_6
-; GFX942-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v8
-; GFX942-NEXT: v_and_b32_e32 v1, -16, v1
-; GFX942-NEXT: v_add3_u32 v2, v0, v1, s16
-; GFX942-NEXT: v_mov_b32_e32 v0, 0xccccdddd
-; GFX942-NEXT: v_mov_b32_e32 v1, 0xaaaabbbb
-; GFX942-NEXT: s_mov_b64 s[8:9], 0
-; GFX942-NEXT: .LBB12_5: ; %memset.pattern-expansion-residual-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s8, s8, 1
-; GFX942-NEXT: s_addc_u32 s9, s9, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[6:7]
-; GFX942-NEXT: buffer_store_dwordx2 v[0:1], v2, s[0:3], 0 offen
-; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-NEXT: v_add_u32_e32 v2, 8, v2
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX942-NEXT: s_cbranch_execnz .LBB12_5
-; GFX942-NEXT: .LBB12_6: ; %Flow1
-; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_pattern_i64_as7_dynlen:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v1
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -2, v8
+; GFX942-SDAG-NEXT: v_and_b32_e32 v6, 1, v8
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB12_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 0xccccdddd
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 0xaaaabbbb
+; GFX942-SDAG-NEXT: v_add_u32_e32 v1, s16, v0
+; GFX942-SDAG-NEXT: s_mov_b64 s[8:9], 0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-SDAG-NEXT: .LBB12_2: ; %memset.pattern-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 2
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen
+; GFX942-SDAG-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX942-SDAG-NEXT: v_add_u32_e32 v1, 32, v1
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB12_2
+; GFX942-SDAG-NEXT: .LBB12_3: ; %Flow3
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB12_6
+; GFX942-SDAG-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
+; GFX942-SDAG-NEXT: v_lshlrev_b32_e32 v1, 3, v8
+; GFX942-SDAG-NEXT: v_and_b32_e32 v1, -16, v1
+; GFX942-SDAG-NEXT: v_add3_u32 v2, v0, v1, s16
+; GFX942-SDAG-NEXT: s_mov_b64 s[8:9], 0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0xccccdddd
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 0xaaaabbbb
+; GFX942-SDAG-NEXT: .LBB12_5: ; %memset.pattern-expansion-residual-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
+; GFX942-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v2, s[0:3], 0 offen
+; GFX942-SDAG-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX942-SDAG-NEXT: v_add_u32_e32 v2, 8, v2
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB12_5
+; GFX942-SDAG-NEXT: .LBB12_6: ; %Flow1
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_pattern_i64_as7_dynlen:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, v1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], 1
+; GFX942-GISEL-NEXT: v_and_b32_e32 v6, 1, v8
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, v2
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v10, vcc, v8, v6
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v11, vcc, 0, v9, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB12_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: s_mov_b32 s12, 0xccccdddd
+; GFX942-GISEL-NEXT: s_mov_b32 s13, 0xaaaabbbb
+; GFX942-GISEL-NEXT: s_mov_b64 s[14:15], s[12:13]
+; GFX942-GISEL-NEXT: v_add_u32_e32 v1, s16, v0
+; GFX942-GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[14:15]
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr10_sgpr11
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr12_sgpr13
+; GFX942-GISEL-NEXT: .LBB12_2: ; %memset.pattern-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 2
+; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
+; GFX942-GISEL-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[12:13], s[12:13], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[14:15], exec, s[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[12:13], s[12:13], s[14:15]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[10:11], s[10:11], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[14:15], exec, s[12:13]
+; GFX942-GISEL-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen
+; GFX942-GISEL-NEXT: v_add_u32_e32 v1, 32, v1
+; GFX942-GISEL-NEXT: s_or_b64 s[10:11], s[10:11], s[14:15]
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB12_2
+; GFX942-GISEL-NEXT: .LBB12_3: ; %Flow3
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB12_6
+; GFX942-GISEL-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
+; GFX942-GISEL-NEXT: v_lshrrev_b64 v[2:3], 1, v[8:9]
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v1, 4, v2
+; GFX942-GISEL-NEXT: v_add3_u32 v2, v0, v1, s16
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0xccccdddd
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 0xaaaabbbb
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr10_sgpr11
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr12_sgpr13
+; GFX942-GISEL-NEXT: .LBB12_5: ; %memset.pattern-expansion-residual-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s6, s6, 1
+; GFX942-GISEL-NEXT: s_addc_u32 s7, s7, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[6:7]
+; GFX942-GISEL-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[12:13], s[12:13], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[14:15], exec, s[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[12:13], s[12:13], s[14:15]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[10:11], s[10:11], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[14:15], exec, s[12:13]
+; GFX942-GISEL-NEXT: buffer_store_dwordx2 v[0:1], v2, s[0:3], 0 offen
+; GFX942-GISEL-NEXT: v_add_u32_e32 v2, 8, v2
+; GFX942-GISEL-NEXT: s_or_b64 s[10:11], s[10:11], s[14:15]
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB12_5
+; GFX942-GISEL-NEXT: .LBB12_6: ; %Flow1
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
%p = getelementptr inbounds i8, ptr addrspace(7) %a, i32 %offset
call void @llvm.experimental.memset.pattern(ptr addrspace(7) align(16) %p, i64 u0xaaaabbbbccccdddd, i64 %len, i1 false)
ret void
}
define void @memset_pattern_i64_as7_dynlen_dynval(ptr addrspace(7) inreg align 16 %a, i32 %offset, i64 %len, i64 %val) {
-; GFX942-LABEL: memset_pattern_i64_as7_dynlen_dynval:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v9, v4
-; GFX942-NEXT: v_mov_b32_e32 v4, v1
-; GFX942-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-NEXT: v_and_b32_e32 v6, -2, v4
-; GFX942-NEXT: v_mov_b32_e32 v8, v3
-; GFX942-NEXT: v_and_b32_e32 v2, 1, v4
-; GFX942-NEXT: v_mov_b32_e32 v3, 0
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GFX942-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB13_3
-; GFX942-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
-; GFX942-NEXT: v_add_u32_e32 v1, s16, v0
-; GFX942-NEXT: v_mov_b32_e32 v10, v8
-; GFX942-NEXT: v_mov_b32_e32 v11, v9
-; GFX942-NEXT: s_mov_b64 s[8:9], 0
-; GFX942-NEXT: .LBB13_2: ; %memset.pattern-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s8, s8, 2
-; GFX942-NEXT: s_addc_u32 s9, s9, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[6:7]
-; GFX942-NEXT: buffer_store_dwordx4 v[8:11], v1, s[0:3], 0 offen
-; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-NEXT: v_add_u32_e32 v1, 32, v1
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX942-NEXT: s_cbranch_execnz .LBB13_2
-; GFX942-NEXT: .LBB13_3: ; %Flow3
-; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
-; GFX942-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB13_6
-; GFX942-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v4
-; GFX942-NEXT: v_and_b32_e32 v1, -16, v1
-; GFX942-NEXT: v_add3_u32 v0, v0, v1, s16
-; GFX942-NEXT: s_mov_b64 s[8:9], 0
-; GFX942-NEXT: .LBB13_5: ; %memset.pattern-expansion-residual-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s8, s8, 1
-; GFX942-NEXT: s_addc_u32 s9, s9, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[2:3]
-; GFX942-NEXT: buffer_store_dwordx2 v[8:9], v0, s[0:3], 0 offen
-; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-NEXT: v_add_u32_e32 v0, 8, v0
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX942-NEXT: s_cbranch_execnz .LBB13_5
-; GFX942-NEXT: .LBB13_6: ; %Flow1
-; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_pattern_i64_as7_dynlen_dynval:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
+; GFX942-SDAG-NEXT: v_and_b32_e32 v6, -2, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v3
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, 1, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB13_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: v_add_u32_e32 v1, s16, v0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v8
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v9
+; GFX942-SDAG-NEXT: s_mov_b64 s[8:9], 0
+; GFX942-SDAG-NEXT: .LBB13_2: ; %memset.pattern-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 2
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v1, s[0:3], 0 offen
+; GFX942-SDAG-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX942-SDAG-NEXT: v_add_u32_e32 v1, 32, v1
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB13_2
+; GFX942-SDAG-NEXT: .LBB13_3: ; %Flow3
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB13_6
+; GFX942-SDAG-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
+; GFX942-SDAG-NEXT: v_lshlrev_b32_e32 v1, 3, v4
+; GFX942-SDAG-NEXT: v_and_b32_e32 v1, -16, v1
+; GFX942-SDAG-NEXT: v_add3_u32 v0, v0, v1, s16
+; GFX942-SDAG-NEXT: s_mov_b64 s[8:9], 0
+; GFX942-SDAG-NEXT: .LBB13_5: ; %memset.pattern-expansion-residual-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
+; GFX942-SDAG-NEXT: buffer_store_dwordx2 v[8:9], v0, s[0:3], 0 offen
+; GFX942-SDAG-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX942-SDAG-NEXT: v_add_u32_e32 v0, 8, v0
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB13_5
+; GFX942-SDAG-NEXT: .LBB13_6: ; %Flow1
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_pattern_i64_as7_dynlen_dynval:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, v1
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, v3
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], 1
+; GFX942-GISEL-NEXT: v_and_b32_e32 v2, 1, v6
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, v4
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v4, vcc, v6, v2
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v5, vcc, 0, v7, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB13_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: v_add_u32_e32 v1, s16, v0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], v[8:9]
+; GFX942-GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr10_sgpr11
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr12_sgpr13
+; GFX942-GISEL-NEXT: .LBB13_2: ; %memset.pattern-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 2
+; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[4:5]
+; GFX942-GISEL-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[12:13], s[12:13], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[14:15], exec, s[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[12:13], s[12:13], s[14:15]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[10:11], s[10:11], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[14:15], exec, s[12:13]
+; GFX942-GISEL-NEXT: buffer_store_dwordx4 v[8:11], v1, s[0:3], 0 offen
+; GFX942-GISEL-NEXT: v_add_u32_e32 v1, 32, v1
+; GFX942-GISEL-NEXT: s_or_b64 s[10:11], s[10:11], s[14:15]
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB13_2
+; GFX942-GISEL-NEXT: .LBB13_3: ; %Flow3
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB13_6
+; GFX942-GISEL-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
+; GFX942-GISEL-NEXT: v_lshrrev_b64 v[4:5], 1, v[6:7]
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v1, 4, v4
+; GFX942-GISEL-NEXT: v_add3_u32 v0, v0, v1, s16
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr10_sgpr11
+; GFX942-GISEL-NEXT: ; implicit-def: $sgpr12_sgpr13
+; GFX942-GISEL-NEXT: .LBB13_5: ; %memset.pattern-expansion-residual-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s6, s6, 1
+; GFX942-GISEL-NEXT: s_addc_u32 s7, s7, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
+; GFX942-GISEL-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[12:13], s[12:13], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[14:15], exec, s[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[12:13], s[12:13], s[14:15]
+; GFX942-GISEL-NEXT: s_andn2_b64 s[10:11], s[10:11], exec
+; GFX942-GISEL-NEXT: s_and_b64 s[14:15], exec, s[12:13]
+; GFX942-GISEL-NEXT: buffer_store_dwordx2 v[8:9], v0, s[0:3], 0 offen
+; GFX942-GISEL-NEXT: v_add_u32_e32 v0, 8, v0
+; GFX942-GISEL-NEXT: s_or_b64 s[10:11], s[10:11], s[14:15]
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB13_5
+; GFX942-GISEL-NEXT: .LBB13_6: ; %Flow1
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
%p = getelementptr inbounds i8, ptr addrspace(7) %a, i32 %offset
call void @llvm.experimental.memset.pattern(ptr addrspace(7) align(16) %p, i64 %val, i64 %len, i1 false)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/multi-divergent-exit-region.ll b/llvm/test/CodeGen/AMDGPU/multi-divergent-exit-region.ll
index 7abd2c29016e2..c154420cfc1d5 100644
--- a/llvm/test/CodeGen/AMDGPU/multi-divergent-exit-region.ll
+++ b/llvm/test/CodeGen/AMDGPU/multi-divergent-exit-region.ll
@@ -10,17 +10,17 @@
; StructurizeCFG.
; IR-LABEL: @multi_divergent_region_exit_ret_ret(
-; IR: %0 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %Pivot)
-; IR: %1 = extractvalue { i1, i64 } %0, 0
-; IR: %2 = extractvalue { i1, i64 } %0, 1
+; IR: %0 = call { i1, i1 } @llvm.amdgcn.if(i1 %Pivot)
+; IR: %1 = extractvalue { i1, i1 } %0, 0
+; IR: %2 = extractvalue { i1, i1 } %0, 1
; IR: br i1 %1, label %LeafBlock1, label %Flow
; IR: Flow:
; IR: %3 = phi i1 [ true, %LeafBlock1 ], [ false, %entry ]
; IR: %4 = phi i1 [ %SwitchLeaf2, %LeafBlock1 ], [ false, %entry ]
-; IR: %5 = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %2)
-; IR: %6 = extractvalue { i1, i64 } %5, 0
-; IR: %7 = extractvalue { i1, i64 } %5, 1
+; IR: %5 = call { i1, i1 } @llvm.amdgcn.else(i1 %2)
+; IR: %6 = extractvalue { i1, i1 } %5, 0
+; IR: %7 = extractvalue { i1, i1 } %5, 1
; IR: br i1 %6, label %LeafBlock, label %Flow1
; IR: LeafBlock:
@@ -31,10 +31,10 @@
; IR: Flow2:
; IR: %8 = phi i1 [ false, %exit1 ], [ %12, %Flow1 ]
-; IR: call void @llvm.amdgcn.end.cf.i64(i64 %16)
-; IR: %9 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %8)
-; IR: %10 = extractvalue { i1, i64 } %9, 0
-; IR: %11 = extractvalue { i1, i64 } %9, 1
+; IR: call void @llvm.amdgcn.end.cf(i1 %16)
+; IR: %9 = call { i1, i1 } @llvm.amdgcn.if(i1 %8)
+; IR: %10 = extractvalue { i1, i1 } %9, 0
+; IR: %11 = extractvalue { i1, i1 } %9, 1
; IR: br i1 %10, label %exit0, label %UnifiedReturnBlock
; IR: exit0:
@@ -44,10 +44,10 @@
; IR: Flow1:
; IR: %12 = phi i1 [ %SwitchLeaf, %LeafBlock ], [ %3, %Flow ]
; IR: %13 = phi i1 [ %SwitchLeaf.inv, %LeafBlock ], [ %4, %Flow ]
-; IR: call void @llvm.amdgcn.end.cf.i64(i64 %7)
-; IR: %14 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %13)
-; IR: %15 = extractvalue { i1, i64 } %14, 0
-; IR: %16 = extractvalue { i1, i64 } %14, 1
+; IR: call void @llvm.amdgcn.end.cf(i1 %7)
+; IR: %14 = call { i1, i1 } @llvm.amdgcn.if(i1 %13)
+; IR: %15 = extractvalue { i1, i1 } %14, 0
+; IR: %16 = extractvalue { i1, i1 } %14, 1
; IR: br i1 %15, label %exit1, label %Flow2
; IR: exit1:
@@ -55,7 +55,7 @@
; IR: br label %Flow2
; IR: UnifiedReturnBlock:
-; IR: call void @llvm.amdgcn.end.cf.i64(i64 %11)
+; IR: call void @llvm.amdgcn.end.cf(i1 %11)
; IR: ret void
@@ -140,13 +140,13 @@ exit1: ; preds = %LeafBlock, %LeafBlock1
}
; IR-LABEL: @multi_divergent_region_exit_unreachable_unreachable(
-; IR: %0 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %Pivot)
+; IR: %0 = call { i1, i1 } @llvm.amdgcn.if(i1 %Pivot)
-; IR: %5 = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %2)
+; IR: %5 = call { i1, i1 } @llvm.amdgcn.else(i1 %2)
; IR: %8 = phi i1 [ false, %exit1 ], [ %12, %Flow1 ]
-; IR: call void @llvm.amdgcn.end.cf.i64(i64 %16)
-; IR: %9 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %8)
+; IR: call void @llvm.amdgcn.end.cf(i1 %16)
+; IR: %9 = call { i1, i1 } @llvm.amdgcn.if(i1 %8)
; IR: br i1 %10, label %exit0, label %UnifiedUnreachableBlock
@@ -202,7 +202,7 @@ exit1: ; preds = %LeafBlock, %LeafBlock1
; IR: {{^}}Flow:
; IR: %3 = phi i1 [ true, %LeafBlock1 ], [ false, %entry ]
; IR: %4 = phi i1 [ %uniform.cond0, %LeafBlock1 ], [ false, %entry ]
-; IR: %5 = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %2)
+; IR: %5 = call { i1, i1 } @llvm.amdgcn.else(i1 %2)
; IR: br i1 %6, label %LeafBlock, label %Flow1
; IR: {{^}}LeafBlock:
@@ -215,8 +215,8 @@ exit1: ; preds = %LeafBlock, %LeafBlock1
; IR: Flow2:
; IR: %8 = phi i1 [ false, %exit1 ], [ %12, %Flow1 ]
-; IR: call void @llvm.amdgcn.end.cf.i64(i64 %16)
-; IR: %9 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %8)
+; IR: call void @llvm.amdgcn.end.cf(i1 %16)
+; IR: %9 = call { i1, i1 } @llvm.amdgcn.if(i1 %8)
; IR: br i1 %10, label %exit0, label %UnifiedReturnBlock
; IR: exit0:
@@ -226,10 +226,10 @@ exit1: ; preds = %LeafBlock, %LeafBlock1
; IR: {{^}}Flow1:
; IR: %12 = phi i1 [ %divergent.cond1, %LeafBlock ], [ %3, %Flow ]
; IR: %13 = phi i1 [ %divergent.cond1.inv, %LeafBlock ], [ %4, %Flow ]
-; IR: call void @llvm.amdgcn.end.cf.i64(i64 %7)
-; IR: %14 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %13)
-; IR: %15 = extractvalue { i1, i64 } %14, 0
-; IR: %16 = extractvalue { i1, i64 } %14, 1
+; IR: call void @llvm.amdgcn.end.cf(i1 %7)
+; IR: %14 = call { i1, i1 } @llvm.amdgcn.if(i1 %13)
+; IR: %15 = extractvalue { i1, i1 } %14, 0
+; IR: %16 = extractvalue { i1, i1 } %14, 1
; IR: br i1 %15, label %exit1, label %Flow2
; IR: exit1:
@@ -237,7 +237,7 @@ exit1: ; preds = %LeafBlock, %LeafBlock1
; IR: br label %Flow2
; IR: UnifiedReturnBlock:
-; IR: call void @llvm.amdgcn.end.cf.i64(i64 %11)
+; IR: call void @llvm.amdgcn.end.cf(i1 %11)
; IR: ret void
define amdgpu_kernel void @multi_exit_region_divergent_ret_uniform_ret(ptr addrspace(1) nocapture %arg0, ptr addrspace(1) nocapture %arg1, ptr addrspace(1) nocapture %arg2, i32 %arg3) #0 {
entry:
@@ -276,17 +276,17 @@ exit1: ; preds = %LeafBlock, %LeafBlock1
}
; IR-LABEL: @multi_exit_region_uniform_ret_divergent_ret(
-; IR: %0 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %Pivot)
+; IR: %0 = call { i1, i1 } @llvm.amdgcn.if(i1 %Pivot)
; IR: br i1 %1, label %LeafBlock1, label %Flow
; IR: Flow:
; IR: %3 = phi i1 [ true, %LeafBlock1 ], [ false, %entry ]
; IR: %4 = phi i1 [ %SwitchLeaf2, %LeafBlock1 ], [ false, %entry ]
-; IR: %5 = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %2)
+; IR: %5 = call { i1, i1 } @llvm.amdgcn.else(i1 %2)
; IR: %8 = phi i1 [ false, %exit1 ], [ %12, %Flow1 ]
-; IR: call void @llvm.amdgcn.end.cf.i64(i64 %16)
-; IR: %9 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %8)
+; IR: call void @llvm.amdgcn.end.cf(i1 %16)
+; IR: %9 = call { i1, i1 } @llvm.amdgcn.if(i1 %8)
define amdgpu_kernel void @multi_exit_region_uniform_ret_divergent_ret(ptr addrspace(1) nocapture %arg0, ptr addrspace(1) nocapture %arg1, ptr addrspace(1) nocapture %arg2, i32 %arg3) #0 {
entry:
@@ -327,11 +327,11 @@ exit1: ; preds = %LeafBlock, %LeafBlock1
; IR-LABEL: @multi_divergent_region_exit_ret_ret_return_value(
; IR: Flow2:
; IR: %8 = phi i1 [ false, %exit1 ], [ %12, %Flow1 ]
-; IR: call void @llvm.amdgcn.end.cf.i64(i64 %16)
+; IR: call void @llvm.amdgcn.end.cf(i1 %16)
; IR: UnifiedReturnBlock:
; IR: %UnifiedRetVal = phi float [ 2.000000e+00, %Flow2 ], [ 1.000000e+00, %exit0 ]
-; IR: call void @llvm.amdgcn.end.cf.i64(i64 %11)
+; IR: call void @llvm.amdgcn.end.cf(i1 %11)
; IR: ret float %UnifiedRetVal
define amdgpu_ps float @multi_divergent_region_exit_ret_ret_return_value(i32 %vgpr) #0 {
entry:
@@ -398,17 +398,17 @@ exit1: ; preds = %LeafBlock, %LeafBlock1
}
; IR-LABEL: @multi_divergent_region_exit_ret_unreachable(
-; IR: %0 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %Pivot)
+; IR: %0 = call { i1, i1 } @llvm.amdgcn.if(i1 %Pivot)
; IR: Flow:
; IR: %3 = phi i1 [ true, %LeafBlock1 ], [ false, %entry ]
; IR: %4 = phi i1 [ %SwitchLeaf2, %LeafBlock1 ], [ false, %entry ]
-; IR: %5 = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %2)
+; IR: %5 = call { i1, i1 } @llvm.amdgcn.else(i1 %2)
; IR: Flow2:
; IR: %8 = phi i1 [ false, %exit1 ], [ %12, %Flow1 ]
-; IR: call void @llvm.amdgcn.end.cf.i64(i64 %16)
-; IR: %9 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %8)
+; IR: call void @llvm.amdgcn.end.cf(i1 %16)
+; IR: %9 = call { i1, i1 } @llvm.amdgcn.if(i1 %8)
; IR: br i1 %10, label %exit0, label %UnifiedReturnBlock
; IR: exit0:
@@ -418,10 +418,10 @@ exit1: ; preds = %LeafBlock, %LeafBlock1
; IR: Flow1:
; IR: %12 = phi i1 [ %SwitchLeaf, %LeafBlock ], [ %3, %Flow ]
; IR: %13 = phi i1 [ %SwitchLeaf.inv, %LeafBlock ], [ %4, %Flow ]
-; IR: call void @llvm.amdgcn.end.cf.i64(i64 %7)
-; IR: %14 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %13)
-; IR: %15 = extractvalue { i1, i64 } %14, 0
-; IR: %16 = extractvalue { i1, i64 } %14, 1
+; IR: call void @llvm.amdgcn.end.cf(i1 %7)
+; IR: %14 = call { i1, i1 } @llvm.amdgcn.if(i1 %13)
+; IR: %15 = extractvalue { i1, i1 } %14, 0
+; IR: %16 = extractvalue { i1, i1 } %14, 1
; IR: br i1 %15, label %exit1, label %Flow2
; IR: exit1:
@@ -430,7 +430,7 @@ exit1: ; preds = %LeafBlock, %LeafBlock1
; IR-NEXT: br label %Flow2
; IR: UnifiedReturnBlock:
-; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 %11)
+; IR-NEXT: call void @llvm.amdgcn.end.cf(i1 %11)
; IR-NEXT: ret void
define amdgpu_kernel void @multi_divergent_region_exit_ret_unreachable(ptr addrspace(1) nocapture %arg0, ptr addrspace(1) nocapture %arg1, ptr addrspace(1) nocapture %arg2) #0 {
entry:
@@ -486,7 +486,7 @@ exit1: ; preds = %LeafBlock, %LeafBlock1
; IR-NEXT: br label %Flow2
; IR: UnifiedReturnBlock: ; preds = %exit0, %Flow2
-; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 %11)
+; IR-NEXT: call void @llvm.amdgcn.end.cf(i1 %11)
; IR-NEXT: ret void
define amdgpu_kernel void @indirect_multi_divergent_region_exit_ret_unreachable(ptr addrspace(1) nocapture %arg0, ptr addrspace(1) nocapture %arg1, ptr addrspace(1) nocapture %arg2) #0 {
entry:
@@ -641,7 +641,7 @@ uniform.ret:
; IR: br i1 %7, label %uniform.endif, label %uniform.ret0
; IR: UnifiedReturnBlock: ; preds = %Flow3, %Flow2
-; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 %5)
+; IR-NEXT: call void @llvm.amdgcn.end.cf(i1 %5)
; IR-NEXT: ret void
define amdgpu_kernel void @uniform_complex_multi_ret_nest_in_divergent_triangle(i32 %arg0) #0 {
entry:
@@ -687,7 +687,7 @@ divergent.ret:
; IR-NEXT: br label %UnifiedReturnBlock
; IR: UnifiedReturnBlock:
-; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64
+; IR-NEXT: call void @llvm.amdgcn.end.cf(i1
; IR-NEXT: ret void
define amdgpu_kernel void @multi_divergent_unreachable_exit(i32 %switch) #0 {
bb:
diff --git a/llvm/test/CodeGen/AMDGPU/multilevel-break.ll b/llvm/test/CodeGen/AMDGPU/multilevel-break.ll
index da012e6ff907d..250e23b091e0f 100644
--- a/llvm/test/CodeGen/AMDGPU/multilevel-break.ll
+++ b/llvm/test/CodeGen/AMDGPU/multilevel-break.ll
@@ -10,32 +10,32 @@ define amdgpu_vs void @multi_else_break(<4 x float> %vec, i32 %ub, i32 %cont) {
; OPT-NEXT: [[MAIN_BODY:.*]]:
; OPT-NEXT: br label %[[LOOP_OUTER:.*]]
; OPT: [[LOOP_OUTER]]:
-; OPT-NEXT: [[PHI_BROKEN2:%.*]] = phi i64 [ [[TMP8:%.*]], %[[FLOW1:.*]] ], [ 0, %[[MAIN_BODY]] ]
+; OPT-NEXT: [[PHI_BROKEN2:%.*]] = phi i1 [ [[TMP8:%.*]], %[[FLOW1:.*]] ], [ false, %[[MAIN_BODY]] ]
; OPT-NEXT: [[TMP43:%.*]] = phi i32 [ 0, %[[MAIN_BODY]] ], [ [[TMP3:%.*]], %[[FLOW1]] ]
; OPT-NEXT: br label %[[LOOP:.*]]
; OPT: [[LOOP]]:
-; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP6:%.*]], %[[FLOW:.*]] ], [ 0, %[[LOOP_OUTER]] ]
+; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i1 [ [[TMP6:%.*]], %[[FLOW:.*]] ], [ false, %[[LOOP_OUTER]] ]
; OPT-NEXT: [[TMP45:%.*]] = phi i32 [ [[TMP43]], %[[LOOP_OUTER]] ], [ [[TMP3]], %[[FLOW]] ]
; OPT-NEXT: [[TMP48:%.*]] = icmp slt i32 [[TMP45]], [[UB]]
-; OPT-NEXT: [[TMP0:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[TMP48]])
-; OPT-NEXT: [[TMP1:%.*]] = extractvalue { i1, i64 } [[TMP0]], 0
-; OPT-NEXT: [[TMP2:%.*]] = extractvalue { i1, i64 } [[TMP0]], 1
+; OPT-NEXT: [[TMP0:%.*]] = call { i1, i1 } @llvm.amdgcn.if(i1 [[TMP48]])
+; OPT-NEXT: [[TMP1:%.*]] = extractvalue { i1, i1 } [[TMP0]], 0
+; OPT-NEXT: [[TMP2:%.*]] = extractvalue { i1, i1 } [[TMP0]], 1
; OPT-NEXT: br i1 [[TMP1]], label %[[ENDIF:.*]], label %[[FLOW]]
; OPT: [[FLOW]]:
; OPT-NEXT: [[TMP3]] = phi i32 [ [[TMP47:%.*]], %[[ENDIF]] ], [ poison, %[[LOOP]] ]
; OPT-NEXT: [[TMP4:%.*]] = phi i1 [ [[TMP51:%.*]], %[[ENDIF]] ], [ true, %[[LOOP]] ]
; OPT-NEXT: [[TMP5:%.*]] = phi i1 [ [[TMP51_INV:%.*]], %[[ENDIF]] ], [ true, %[[LOOP]] ]
-; OPT-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP2]])
-; OPT-NEXT: [[TMP6]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[TMP5]], i64 [[PHI_BROKEN]])
-; OPT-NEXT: [[TMP7:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP6]])
+; OPT-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP2]])
+; OPT-NEXT: [[TMP6]] = call i1 @llvm.amdgcn.if.break(i1 [[TMP5]], i1 [[PHI_BROKEN]])
+; OPT-NEXT: [[TMP7:%.*]] = call i1 @llvm.amdgcn.loop(i1 [[TMP6]])
; OPT-NEXT: br i1 [[TMP7]], label %[[FLOW1]], label %[[LOOP]]
; OPT: [[FLOW1]]:
-; OPT-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP6]])
-; OPT-NEXT: [[TMP8]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[TMP4]], i64 [[PHI_BROKEN2]])
-; OPT-NEXT: [[TMP9:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP8]])
+; OPT-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP6]])
+; OPT-NEXT: [[TMP8]] = call i1 @llvm.amdgcn.if.break(i1 [[TMP4]], i1 [[PHI_BROKEN2]])
+; OPT-NEXT: [[TMP9:%.*]] = call i1 @llvm.amdgcn.loop(i1 [[TMP8]])
; OPT-NEXT: br i1 [[TMP9]], label %[[IF:.*]], label %[[LOOP_OUTER]]
; OPT: [[IF]]:
-; OPT-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP8]])
+; OPT-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP8]])
; OPT-NEXT: ret void
; OPT: [[ENDIF]]:
; OPT-NEXT: [[TMP47]] = add i32 [[TMP45]], 1
@@ -121,7 +121,7 @@ define amdgpu_kernel void @multi_if_break_loop(i32 %arg) #0 {
; OPT-NEXT: [[TMP:%.*]] = sub i32 [[ID]], [[ARG]]
; OPT-NEXT: br label %[[BB1:.*]]
; OPT: [[BB1]]:
-; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP4:%.*]], %[[FLOW4:.*]] ], [ 0, %[[BB]] ]
+; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i1 [ [[TMP4:%.*]], %[[FLOW4:.*]] ], [ false, %[[BB]] ]
; OPT-NEXT: [[LSR_IV:%.*]] = phi i32 [ poison, %[[BB]] ], [ [[TMP2:%.*]], %[[FLOW4]] ]
; OPT-NEXT: [[TMP2]] = add i32 [[LSR_IV]], 1
; OPT-NEXT: [[CMP0:%.*]] = icmp slt i32 [[TMP2]], 0
@@ -141,8 +141,8 @@ define amdgpu_kernel void @multi_if_break_loop(i32 %arg) #0 {
; OPT-NEXT: br i1 [[SWITCHLEAF]], label %[[CASE0:.*]], label %[[FLOW5:.*]]
; OPT: [[FLOW4]]:
; OPT-NEXT: [[TMP3:%.*]] = phi i1 [ [[TMP10:%.*]], %[[FLOW5]] ], [ [[TMP7:%.*]], %[[FLOW]] ]
-; OPT-NEXT: [[TMP4]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[TMP3]], i64 [[PHI_BROKEN]])
-; OPT-NEXT: [[TMP5:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP4]])
+; OPT-NEXT: [[TMP4]] = call i1 @llvm.amdgcn.if.break(i1 [[TMP3]], i1 [[PHI_BROKEN]])
+; OPT-NEXT: [[TMP5:%.*]] = call i1 @llvm.amdgcn.loop(i1 [[TMP4]])
; OPT-NEXT: br i1 [[TMP5]], label %[[BB9:.*]], label %[[BB1]]
; OPT: [[CASE0]]:
; OPT-NEXT: [[LOAD1:%.*]] = load volatile i32, ptr addrspace(1) poison, align 4
@@ -160,7 +160,7 @@ define amdgpu_kernel void @multi_if_break_loop(i32 %arg) #0 {
; OPT-NEXT: [[TMP10]] = phi i1 [ [[CMP1]], %[[CASE0]] ], [ [[TMP7]], %[[LEAFBLOCK]] ]
; OPT-NEXT: br label %[[FLOW4]]
; OPT: [[BB9]]:
-; OPT-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP4]])
+; OPT-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP4]])
; OPT-NEXT: ret void
;
; GCN-LABEL: multi_if_break_loop:
diff --git a/llvm/test/CodeGen/AMDGPU/nested-loop-conditions.ll b/llvm/test/CodeGen/AMDGPU/nested-loop-conditions.ll
index 9e88a62925e50..592adbe8660e9 100644
--- a/llvm/test/CodeGen/AMDGPU/nested-loop-conditions.ll
+++ b/llvm/test/CodeGen/AMDGPU/nested-loop-conditions.ll
@@ -57,25 +57,25 @@ define amdgpu_kernel void @reduced_nested_loop_conditions(ptr addrspace(3) captu
; IR: [[BB4]]:
; IR-NEXT: br label %[[FLOW:.*]]
; IR: [[BB5]]:
-; IR-NEXT: [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP6:%.*]], %[[BB10:.*]] ], [ 0, %[[BB]] ]
+; IR-NEXT: [[PHI_BROKEN:%.*]] = phi i1 [ [[TMP6:%.*]], %[[BB10:.*]] ], [ false, %[[BB]] ]
; IR-NEXT: [[MY_TMP6:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[TMP5:%.*]], %[[BB10]] ]
; IR-NEXT: [[MY_TMP7:%.*]] = icmp eq i32 [[MY_TMP6]], 1
-; IR-NEXT: [[TMP0:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[MY_TMP7]])
-; IR-NEXT: [[TMP1:%.*]] = extractvalue { i1, i64 } [[TMP0]], 0
-; IR-NEXT: [[TMP2:%.*]] = extractvalue { i1, i64 } [[TMP0]], 1
+; IR-NEXT: [[TMP0:%.*]] = call { i1, i1 } @llvm.amdgcn.if(i1 [[MY_TMP7]])
+; IR-NEXT: [[TMP1:%.*]] = extractvalue { i1, i1 } [[TMP0]], 0
+; IR-NEXT: [[TMP2:%.*]] = extractvalue { i1, i1 } [[TMP0]], 1
; IR-NEXT: br i1 [[TMP1]], label %[[BB8:.*]], label %[[FLOW]]
; IR: [[BB8]]:
; IR-NEXT: br label %[[BB13]]
; IR: [[BB9:.*]]:
; IR-NEXT: br i1 false, label %[[BB3]], label %[[BB9]]
; IR: [[BB10]]:
-; IR-NEXT: [[TMP3:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP6]])
+; IR-NEXT: [[TMP3:%.*]] = call i1 @llvm.amdgcn.loop(i1 [[TMP6]])
; IR-NEXT: br i1 [[TMP3]], label %[[BB23:.*]], label %[[BB5]]
; IR: [[FLOW]]:
; IR-NEXT: [[TMP4:%.*]] = phi i1 [ [[MY_TMP22:%.*]], %[[BB4]] ], [ true, %[[BB5]] ]
; IR-NEXT: [[TMP5]] = phi i32 [ [[MY_TMP21:%.*]], %[[BB4]] ], [ poison, %[[BB5]] ]
-; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP2]])
-; IR-NEXT: [[TMP6]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[TMP4]], i64 [[PHI_BROKEN]])
+; IR-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP2]])
+; IR-NEXT: [[TMP6]] = call i1 @llvm.amdgcn.if.break(i1 [[TMP4]], i1 [[PHI_BROKEN]])
; IR-NEXT: br label %[[BB10]]
; IR: [[BB13]]:
; IR-NEXT: [[MY_TMP14:%.*]] = phi i1 [ [[MY_TMP22]], %[[BB3]] ], [ true, %[[BB8]] ]
@@ -91,9 +91,8 @@ define amdgpu_kernel void @reduced_nested_loop_conditions(ptr addrspace(3) captu
; IR-NEXT: [[MY_TMP22]] = phi i1 [ false, %[[BB16]] ], [ [[MY_TMP14]], %[[BB13]] ]
; IR-NEXT: br label %[[BB9]]
; IR: [[BB23]]:
-; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP6]])
+; IR-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP6]])
; IR-NEXT: ret void
-;
bb:
%my.tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%my.tmp1 = getelementptr inbounds i64, ptr addrspace(3) %arg, i32 %my.tmp
@@ -204,33 +203,33 @@ define amdgpu_kernel void @nested_loop_conditions(ptr addrspace(1) captures(none
; IR-NEXT: [[MY_TMP1033:%.*]] = extractelement <4 x i32> [[MY_TMP932]], i64 0
; IR-NEXT: br label %[[BB14:.*]]
; IR: [[FLOW3:.*]]:
-; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP20:%.*]])
-; IR-NEXT: [[TMP0:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[TMP14:%.*]])
-; IR-NEXT: [[TMP1:%.*]] = extractvalue { i1, i64 } [[TMP0]], 0
-; IR-NEXT: [[TMP2:%.*]] = extractvalue { i1, i64 } [[TMP0]], 1
+; IR-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP20:%.*]])
+; IR-NEXT: [[TMP0:%.*]] = call { i1, i1 } @llvm.amdgcn.if(i1 [[TMP14:%.*]])
+; IR-NEXT: [[TMP1:%.*]] = extractvalue { i1, i1 } [[TMP0]], 0
+; IR-NEXT: [[TMP2:%.*]] = extractvalue { i1, i1 } [[TMP0]], 1
; IR-NEXT: br i1 [[TMP1]], label %[[BB4_BB13_CRIT_EDGE:.*]], label %[[FLOW4:.*]]
; IR: [[BB4_BB13_CRIT_EDGE]]:
; IR-NEXT: br label %[[FLOW4]]
; IR: [[FLOW4]]:
; IR-NEXT: [[TMP3:%.*]] = phi i1 [ true, %[[BB4_BB13_CRIT_EDGE]] ], [ false, %[[FLOW3]] ]
-; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP2]])
+; IR-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP2]])
; IR-NEXT: br label %[[FLOW]]
; IR: [[BB13:.*]]:
; IR-NEXT: br label %[[BB31:.*]]
; IR: [[FLOW]]:
; IR-NEXT: [[TMP4:%.*]] = phi i1 [ [[TMP3]], %[[FLOW4]] ], [ true, %[[BB]] ]
-; IR-NEXT: [[TMP5:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = extractvalue { i1, i64 } [[TMP5]], 0
-; IR-NEXT: [[TMP7:%.*]] = extractvalue { i1, i64 } [[TMP5]], 1
+; IR-NEXT: [[TMP5:%.*]] = call { i1, i1 } @llvm.amdgcn.if(i1 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = extractvalue { i1, i1 } [[TMP5]], 0
+; IR-NEXT: [[TMP7:%.*]] = extractvalue { i1, i1 } [[TMP5]], 1
; IR-NEXT: br i1 [[TMP6]], label %[[BB13]], label %[[BB31]]
; IR: [[BB14]]:
-; IR-NEXT: [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP16:%.*]], %[[FLOW1:.*]] ], [ 0, %[[BB14_LR_PH]] ]
+; IR-NEXT: [[PHI_BROKEN:%.*]] = phi i1 [ [[TMP16:%.*]], %[[FLOW1:.*]] ], [ false, %[[BB14_LR_PH]] ]
; IR-NEXT: [[MY_TMP1037:%.*]] = phi i32 [ [[MY_TMP1033]], %[[BB14_LR_PH]] ], [ [[TMP12:%.*]], %[[FLOW1]] ]
; IR-NEXT: [[MY_TMP936:%.*]] = phi <4 x i32> [ [[MY_TMP932]], %[[BB14_LR_PH]] ], [ [[TMP11:%.*]], %[[FLOW1]] ]
; IR-NEXT: [[MY_TMP15:%.*]] = icmp eq i32 [[MY_TMP1037]], 1
-; IR-NEXT: [[TMP8:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[MY_TMP15]])
-; IR-NEXT: [[TMP9:%.*]] = extractvalue { i1, i64 } [[TMP8]], 0
-; IR-NEXT: [[TMP10:%.*]] = extractvalue { i1, i64 } [[TMP8]], 1
+; IR-NEXT: [[TMP8:%.*]] = call { i1, i1 } @llvm.amdgcn.if(i1 [[MY_TMP15]])
+; IR-NEXT: [[TMP9:%.*]] = extractvalue { i1, i1 } [[TMP8]], 0
+; IR-NEXT: [[TMP10:%.*]] = extractvalue { i1, i1 } [[TMP8]], 1
; IR-NEXT: br i1 [[TMP9]], label %[[BB16:.*]], label %[[FLOW1]]
; IR: [[BB16]]:
; IR-NEXT: [[MY_TMP17:%.*]] = bitcast i64 [[MY_TMP3]] to <2 x i32>
@@ -241,9 +240,9 @@ define amdgpu_kernel void @nested_loop_conditions(ptr addrspace(1) captures(none
; IR-NEXT: [[TMP13:%.*]] = phi i1 [ [[MY_TMP12:%.*]], %[[BB21]] ], [ true, %[[BB14]] ]
; IR-NEXT: [[TMP14]] = phi i1 [ [[MY_TMP12]], %[[BB21]] ], [ false, %[[BB14]] ]
; IR-NEXT: [[TMP15:%.*]] = phi i1 [ false, %[[BB21]] ], [ true, %[[BB14]] ]
-; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP10]])
-; IR-NEXT: [[TMP16]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[TMP13]], i64 [[PHI_BROKEN]])
-; IR-NEXT: [[TMP17:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP16]])
+; IR-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP10]])
+; IR-NEXT: [[TMP16]] = call i1 @llvm.amdgcn.if.break(i1 [[TMP13]], i1 [[PHI_BROKEN]])
+; IR-NEXT: [[TMP17:%.*]] = call i1 @llvm.amdgcn.loop(i1 [[TMP16]])
; IR-NEXT: br i1 [[TMP17]], label %[[FLOW2:.*]], label %[[BB14]]
; IR: [[BB18]]:
; IR-NEXT: [[MY_TMP19:%.*]] = load volatile i32, ptr addrspace(1) poison, align 4
@@ -267,18 +266,17 @@ define amdgpu_kernel void @nested_loop_conditions(ptr addrspace(1) captures(none
; IR-NEXT: [[MY_TMP12]] = icmp sge i32 [[MY_TMP11]], 9
; IR-NEXT: br label %[[FLOW1]]
; IR: [[FLOW2]]:
-; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP16]])
-; IR-NEXT: [[TMP18:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[TMP15]])
-; IR-NEXT: [[TMP19:%.*]] = extractvalue { i1, i64 } [[TMP18]], 0
-; IR-NEXT: [[TMP20]] = extractvalue { i1, i64 } [[TMP18]], 1
+; IR-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP16]])
+; IR-NEXT: [[TMP18:%.*]] = call { i1, i1 } @llvm.amdgcn.if(i1 [[TMP15]])
+; IR-NEXT: [[TMP19:%.*]] = extractvalue { i1, i1 } [[TMP18]], 0
+; IR-NEXT: [[TMP20]] = extractvalue { i1, i1 } [[TMP18]], 1
; IR-NEXT: br i1 [[TMP19]], label %[[BB31_LOOPEXIT:.*]], label %[[FLOW3]]
; IR: [[BB31_LOOPEXIT]]:
; IR-NEXT: br label %[[FLOW3]]
; IR: [[BB31]]:
-; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP7]])
+; IR-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP7]])
; IR-NEXT: store volatile i32 0, ptr addrspace(1) poison, align 4
; IR-NEXT: ret void
-;
bb:
%my.tmp1134 = load volatile i32, ptr addrspace(1) poison
%my.tmp1235 = icmp slt i32 %my.tmp1134, 9
diff --git a/llvm/test/CodeGen/AMDGPU/rem_i128.ll b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
index 23b379edae8b1..493fb3236e073 100644
--- a/llvm/test/CodeGen/AMDGPU/rem_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
@@ -97,7 +97,7 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_sub_u32_e32 v9, 64, v11
; GFX9-NEXT: v_lshlrev_b64 v[6:7], v11, v[2:3]
; GFX9-NEXT: v_lshrrev_b64 v[9:10], v9, v[0:1]
-; GFX9-NEXT: s_xor_b64 s[8:9], vcc, -1
+; GFX9-NEXT: s_xor_b64 s[10:11], vcc, -1
; GFX9-NEXT: v_or_b32_e32 v9, v6, v9
; GFX9-NEXT: v_sub_u32_e32 v6, 63, v8
; GFX9-NEXT: v_or_b32_e32 v10, v7, v10
@@ -107,14 +107,15 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_lshlrev_b64 v[8:9], v11, v[0:1]
; GFX9-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc
; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v11
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v3, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v2, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e32 v9, 0, v9, vcc
; GFX9-NEXT: v_mov_b32_e32 v10, 0
; GFX9-NEXT: v_mov_b32_e32 v11, 0
; GFX9-NEXT: v_cndmask_b32_e32 v8, 0, v8, vcc
-; GFX9-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GFX9-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GFX9-NEXT: s_and_saveexec_b64 s[4:5], s[10:11]
+; GFX9-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB0_5
; GFX9-NEXT: ; %bb.2: ; %udiv-preheader
; GFX9-NEXT: v_sub_u32_e32 v12, 64, v24
@@ -139,7 +140,6 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_addc_co_u32_e32 v31, vcc, -1, v5, vcc
; GFX9-NEXT: v_mov_b32_e32 v12, 0
; GFX9-NEXT: v_mov_b32_e32 v13, 0
-; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: v_mov_b32_e32 v18, 0
; GFX9-NEXT: v_mov_b32_e32 v19, 0
; GFX9-NEXT: v_mov_b32_e32 v11, 0
@@ -179,15 +179,15 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_or3_b32 v6, v6, v10, v12
; GFX9-NEXT: v_and_b32_e32 v10, 1, v32
; GFX9-NEXT: v_or3_b32 v7, v7, 0, v13
-; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX9-NEXT: v_mov_b32_e32 v19, v11
; GFX9-NEXT: v_mov_b32_e32 v18, v10
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX9-NEXT: s_cbranch_execnz .LBB0_3
; GFX9-NEXT: ; %bb.4: ; %Flow
-; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: .LBB0_5: ; %Flow2
; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX9-NEXT: .LBB0_5: ; %Flow2
+; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX9-NEXT: v_lshlrev_b64 v[6:7], 1, v[6:7]
; GFX9-NEXT: v_lshrrev_b32_e32 v12, 31, v9
; GFX9-NEXT: v_lshlrev_b64 v[8:9], 1, v[8:9]
@@ -991,6 +991,7 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_mov_b64 s[4:5], 0
; GFX9-O0-NEXT: v_writelane_b32 v31, s4, 8
; GFX9-O0-NEXT: v_writelane_b32 v31, s5, 9
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
@@ -1523,7 +1524,7 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_sub_u32_e32 v11, 64, v13
; GFX9-NEXT: v_lshlrev_b64 v[8:9], v13, v[2:3]
; GFX9-NEXT: v_lshrrev_b64 v[11:12], v11, v[0:1]
-; GFX9-NEXT: s_xor_b64 s[8:9], vcc, -1
+; GFX9-NEXT: s_xor_b64 s[10:11], vcc, -1
; GFX9-NEXT: v_or_b32_e32 v11, v8, v11
; GFX9-NEXT: v_sub_u32_e32 v8, 63, v10
; GFX9-NEXT: v_or_b32_e32 v12, v9, v12
@@ -1533,14 +1534,15 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_lshlrev_b64 v[10:11], v13, v[0:1]
; GFX9-NEXT: v_cndmask_b32_e32 v9, v9, v12, vcc
; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v13
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
; GFX9-NEXT: v_cndmask_b32_e64 v9, v9, v3, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e64 v8, v8, v2, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e32 v11, 0, v11, vcc
; GFX9-NEXT: v_mov_b32_e32 v12, 0
; GFX9-NEXT: v_mov_b32_e32 v13, 0
; GFX9-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc
-; GFX9-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GFX9-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GFX9-NEXT: s_and_saveexec_b64 s[4:5], s[10:11]
+; GFX9-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB1_5
; GFX9-NEXT: ; %bb.2: ; %udiv-preheader
; GFX9-NEXT: v_sub_u32_e32 v14, 64, v22
@@ -1565,7 +1567,6 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_addc_co_u32_e32 v29, vcc, -1, v7, vcc
; GFX9-NEXT: v_mov_b32_e32 v14, 0
; GFX9-NEXT: v_mov_b32_e32 v15, 0
-; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: v_mov_b32_e32 v20, 0
; GFX9-NEXT: v_mov_b32_e32 v21, 0
; GFX9-NEXT: v_mov_b32_e32 v13, 0
@@ -1605,15 +1606,15 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_or3_b32 v8, v8, v12, v14
; GFX9-NEXT: v_and_b32_e32 v12, 1, v30
; GFX9-NEXT: v_or3_b32 v9, v9, 0, v15
-; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX9-NEXT: v_mov_b32_e32 v21, v13
; GFX9-NEXT: v_mov_b32_e32 v20, v12
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX9-NEXT: s_cbranch_execnz .LBB1_3
; GFX9-NEXT: ; %bb.4: ; %Flow
-; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: .LBB1_5: ; %Flow2
; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX9-NEXT: .LBB1_5: ; %Flow2
+; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX9-NEXT: v_lshlrev_b64 v[8:9], 1, v[8:9]
; GFX9-NEXT: v_lshrrev_b32_e32 v14, 31, v11
; GFX9-NEXT: v_lshlrev_b64 v[10:11], 1, v[10:11]
@@ -2330,6 +2331,7 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_mov_b64 s[4:5], 0
; GFX9-O0-NEXT: v_writelane_b32 v31, s4, 8
; GFX9-O0-NEXT: v_writelane_b32 v31, s5, 9
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
diff --git a/llvm/test/CodeGen/AMDGPU/sdiv64.ll b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
index 9554a03942579..19ddaa69f4c69 100644
--- a/llvm/test/CodeGen/AMDGPU/sdiv64.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
@@ -434,9 +434,9 @@ define i64 @v_test_sdiv(i64 %x, i64 %y) {
; GCN-IR-NEXT: v_not_b32_e32 v4, v8
; GCN-IR-NEXT: v_add_i32_e32 v16, vcc, v4, v9
; GCN-IR-NEXT: v_addc_u32_e64 v17, s[8:9], -1, 0, vcc
-; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v8, 0
; GCN-IR-NEXT: v_mov_b32_e32 v9, 0
+; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: .LBB1_3: ; %udiv-do-while
; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -1838,9 +1838,9 @@ define i64 @v_test_sdiv_k_num_i64(i64 %x) {
; GCN-IR-NEXT: v_sub_i32_e32 v14, vcc, 58, v8
; GCN-IR-NEXT: v_lshr_b64 v[6:7], 24, v6
; GCN-IR-NEXT: v_subb_u32_e64 v15, s[8:9], 0, 0, vcc
-; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v8, 0
; GCN-IR-NEXT: v_mov_b32_e32 v9, 0
+; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: .LBB12_3: ; %udiv-do-while
; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -2032,9 +2032,9 @@ define i64 @v_test_sdiv_pow2_k_num_i64(i64 %x) {
; GCN-IR-NEXT: v_sub_i32_e32 v14, vcc, 47, v8
; GCN-IR-NEXT: v_lshr_b64 v[6:7], s[8:9], v6
; GCN-IR-NEXT: v_subb_u32_e64 v15, s[8:9], 0, 0, vcc
-; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v8, 0
; GCN-IR-NEXT: v_mov_b32_e32 v9, 0
+; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: .LBB13_3: ; %udiv-do-while
; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -2128,9 +2128,9 @@ define i64 @v_test_sdiv_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_add_i32_e32 v10, vcc, 0xffffffcf, v6
; GCN-IR-NEXT: v_lshr_b64 v[4:5], v[4:5], v7
; GCN-IR-NEXT: v_addc_u32_e64 v11, s[8:9], 0, -1, vcc
-; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v6, 0
; GCN-IR-NEXT: v_mov_b32_e32 v7, 0
+; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v3, 0
; GCN-IR-NEXT: s_movk_i32 s10, 0x7fff
; GCN-IR-NEXT: .LBB14_3: ; %udiv-do-while
@@ -2516,9 +2516,9 @@ define i64 @v_test_sdiv24_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_add_i32_e32 v10, vcc, 0xffffffcf, v6
; GCN-IR-NEXT: v_lshr_b64 v[4:5], v[4:5], v7
; GCN-IR-NEXT: v_addc_u32_e64 v11, s[8:9], 0, -1, vcc
-; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v6, 0
; GCN-IR-NEXT: v_mov_b32_e32 v7, 0
+; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v3, 0
; GCN-IR-NEXT: s_movk_i32 s10, 0x7fff
; GCN-IR-NEXT: .LBB19_3: ; %udiv-do-while
diff --git a/llvm/test/CodeGen/AMDGPU/si-annotate-cf-unreachable.ll b/llvm/test/CodeGen/AMDGPU/si-annotate-cf-unreachable.ll
index 1bbb8f0e28973..d16bec923c9dd 100644
--- a/llvm/test/CodeGen/AMDGPU/si-annotate-cf-unreachable.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-annotate-cf-unreachable.ll
@@ -4,7 +4,7 @@
; OPT-LABEL: @annotate_unreachable(
-; OPT: call { i1, i64 } @llvm.amdgcn.if.i64(
+; OPT: call { i1, i1 } @llvm.amdgcn.if(
; OPT-NOT: call void @llvm.amdgcn.end.cf
diff --git a/llvm/test/CodeGen/AMDGPU/si-annotate-dbg-info.ll b/llvm/test/CodeGen/AMDGPU/si-annotate-dbg-info.ll
index f163c63f074ae..b26370e8f410c 100644
--- a/llvm/test/CodeGen/AMDGPU/si-annotate-dbg-info.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-annotate-dbg-info.ll
@@ -6,14 +6,14 @@ define amdgpu_ps i32 @if_else(i32 %0) !dbg !5 {
; OPT-SAME: i32 [[TMP0:%.*]]) !dbg [[DBG5:![0-9]+]] {
; OPT-NEXT: [[C:%.*]] = icmp ne i32 [[TMP0]], 0, !dbg [[DBG13:![0-9]+]]
; OPT-NEXT: #dbg_value(i1 [[C]], [[META9:![0-9]+]], !DIExpression(), [[DBG13]])
-; OPT-NEXT: [[TMP2:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[C]]), !dbg [[DBG14:![0-9]+]]
-; OPT-NEXT: [[TMP3:%.*]] = extractvalue { i1, i64 } [[TMP2]], 0, !dbg [[DBG14]]
-; OPT-NEXT: [[TMP4:%.*]] = extractvalue { i1, i64 } [[TMP2]], 1, !dbg [[DBG14]]
+; OPT-NEXT: [[TMP2:%.*]] = call { i1, i1 } @llvm.amdgcn.if(i1 [[C]]), !dbg [[DBG14:![0-9]+]]
+; OPT-NEXT: [[TMP3:%.*]] = extractvalue { i1, i1 } [[TMP2]], 0, !dbg [[DBG14]]
+; OPT-NEXT: [[TMP4:%.*]] = extractvalue { i1, i1 } [[TMP2]], 1, !dbg [[DBG14]]
; OPT-NEXT: br i1 [[TMP3]], label [[FALSE:%.*]], label [[FLOW:%.*]], !dbg [[DBG14]]
; OPT: Flow:
-; OPT-NEXT: [[TMP5:%.*]] = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 [[TMP4]])
-; OPT-NEXT: [[TMP6:%.*]] = extractvalue { i1, i64 } [[TMP5]], 0
-; OPT-NEXT: [[TMP8:%.*]] = extractvalue { i1, i64 } [[TMP5]], 1
+; OPT-NEXT: [[TMP5:%.*]] = call { i1, i1 } @llvm.amdgcn.else(i1 [[TMP4]])
+; OPT-NEXT: [[TMP6:%.*]] = extractvalue { i1, i1 } [[TMP5]], 0
+; OPT-NEXT: [[TMP7:%.*]] = extractvalue { i1, i1 } [[TMP5]], 1
; OPT-NEXT: br i1 [[TMP6]], label [[TRUE:%.*]], label [[EXIT:%.*]]
; OPT: true:
; OPT-NEXT: br label [[EXIT]], !dbg [[DBG15:![0-9]+]]
@@ -21,7 +21,7 @@ define amdgpu_ps i32 @if_else(i32 %0) !dbg !5 {
; OPT-NEXT: br label [[FLOW]], !dbg [[DBG16:![0-9]+]]
; OPT: exit:
; OPT-NEXT: [[RET:%.*]] = phi i32 [ 33, [[FLOW]] ], [ 42, [[TRUE]] ], !dbg [[DBG17:![0-9]+]]
-; OPT-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP8]])
+; OPT-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP7]])
; OPT-NEXT: #dbg_value(i32 [[RET]], [[META11:![0-9]+]], !DIExpression(), [[DBG17]])
; OPT-NEXT: ret i32 [[RET]], !dbg [[DBG18:![0-9]+]]
;
@@ -47,14 +47,14 @@ define amdgpu_ps void @loop_if_break(i32 %n) !dbg !19 {
; OPT-NEXT: entry:
; OPT-NEXT: br label [[LOOP:%.*]], !dbg [[DBG24:![0-9]+]]
; OPT: loop:
-; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP5:%.*]], [[FLOW:%.*]] ], [ 0, [[ENTRY:%.*]] ]
+; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i1 [ [[TMP5:%.*]], [[FLOW:%.*]] ], [ false, [[ENTRY:%.*]] ]
; OPT-NEXT: [[I:%.*]] = phi i32 [ [[N]], [[ENTRY]] ], [ [[TMP3:%.*]], [[FLOW]] ], !dbg [[DBG25:![0-9]+]]
; OPT-NEXT: #dbg_value(i32 [[I]], [[META21:![0-9]+]], !DIExpression(), [[DBG25]])
; OPT-NEXT: [[C:%.*]] = icmp ugt i32 [[I]], 0, !dbg [[DBG26:![0-9]+]]
; OPT-NEXT: #dbg_value(i1 [[C]], [[META22:![0-9]+]], !DIExpression(), [[DBG26]])
-; OPT-NEXT: [[TMP0:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[C]]), !dbg [[DBG27:![0-9]+]]
-; OPT-NEXT: [[TMP1:%.*]] = extractvalue { i1, i64 } [[TMP0]], 0, !dbg [[DBG27]]
-; OPT-NEXT: [[TMP2:%.*]] = extractvalue { i1, i64 } [[TMP0]], 1, !dbg [[DBG27]]
+; OPT-NEXT: [[TMP0:%.*]] = call { i1, i1 } @llvm.amdgcn.if(i1 [[C]]), !dbg [[DBG27:![0-9]+]]
+; OPT-NEXT: [[TMP1:%.*]] = extractvalue { i1, i1 } [[TMP0]], 0, !dbg [[DBG27]]
+; OPT-NEXT: [[TMP2:%.*]] = extractvalue { i1, i1 } [[TMP0]], 1, !dbg [[DBG27]]
; OPT-NEXT: br i1 [[TMP1]], label [[LOOP_BODY:%.*]], label [[FLOW]], !dbg [[DBG27]]
; OPT: loop_body:
; OPT-NEXT: [[I_NEXT:%.*]] = sub i32 [[I]], 1, !dbg [[DBG28:![0-9]+]]
@@ -63,12 +63,12 @@ define amdgpu_ps void @loop_if_break(i32 %n) !dbg !19 {
; OPT: Flow:
; OPT-NEXT: [[TMP3]] = phi i32 [ [[I_NEXT]], [[LOOP_BODY]] ], [ poison, [[LOOP]] ]
; OPT-NEXT: [[TMP4:%.*]] = phi i1 [ false, [[LOOP_BODY]] ], [ true, [[LOOP]] ]
-; OPT-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP2]])
-; OPT-NEXT: [[TMP5]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[TMP4]], i64 [[PHI_BROKEN]])
-; OPT-NEXT: [[TMP6:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP5]])
+; OPT-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP2]])
+; OPT-NEXT: [[TMP5]] = call i1 @llvm.amdgcn.if.break(i1 [[TMP4]], i1 [[PHI_BROKEN]])
+; OPT-NEXT: [[TMP6:%.*]] = call i1 @llvm.amdgcn.loop(i1 [[TMP5]])
; OPT-NEXT: br i1 [[TMP6]], label [[EXIT:%.*]], label [[LOOP]]
; OPT: exit:
-; OPT-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP5]])
+; OPT-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP5]])
; OPT-NEXT: ret void, !dbg [[DBG30:![0-9]+]]
;
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/si-annotate-loop-i1-lane-mask.ll b/llvm/test/CodeGen/AMDGPU/si-annotate-loop-i1-lane-mask.ll
index 90624e7cc9f33..0c66b226bcd34 100644
--- a/llvm/test/CodeGen/AMDGPU/si-annotate-loop-i1-lane-mask.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-annotate-loop-i1-lane-mask.ll
@@ -12,15 +12,15 @@ define amdgpu_kernel void @divergent_loop(ptr addrspace(1) %p) {
; OPT-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
; OPT-NEXT: br label [[LOOP:%.*]]
; OPT: loop:
-; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP0:%.*]], [[LOOP]] ], [ 0, [[ENTRY:%.*]] ]
+; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i1 [ [[TMP0:%.*]], [[LOOP]] ], [ false, [[ENTRY:%.*]] ]
; OPT-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
; OPT-NEXT: [[I_NEXT]] = add i32 [[I]], 1
; OPT-NEXT: [[COND:%.*]] = icmp sge i32 [[I]], [[TID]]
-; OPT-NEXT: [[TMP0]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[COND]], i64 [[PHI_BROKEN]])
-; OPT-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP0]])
+; OPT-NEXT: [[TMP0]] = call i1 @llvm.amdgcn.if.break(i1 [[COND]], i1 [[PHI_BROKEN]])
+; OPT-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.loop(i1 [[TMP0]])
; OPT-NEXT: br i1 [[TMP1]], label [[EXIT:%.*]], label [[LOOP]]
; OPT: exit:
-; OPT-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP0]])
+; OPT-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP0]])
; OPT-NEXT: store i32 [[I]], ptr addrspace(1) [[P:%.*]], align 4
; OPT-NEXT: ret void
;
diff --git a/llvm/test/CodeGen/AMDGPU/si-annotatecfg-multiple-backedges.ll b/llvm/test/CodeGen/AMDGPU/si-annotatecfg-multiple-backedges.ll
index 0edd9f4cd6b4f..eb80797f33103 100644
--- a/llvm/test/CodeGen/AMDGPU/si-annotatecfg-multiple-backedges.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-annotatecfg-multiple-backedges.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --prefix-filecheck-ir-name _
; RUN: opt -mtriple=amdgcn-- -S -structurizecfg -si-annotate-control-flow %s | FileCheck -check-prefix=OPT %s
; This test is designed to check that the backedge from a prior block won't
@@ -12,22 +12,22 @@ define amdgpu_kernel void @multiple_backedges(i32 %arg, ptr %arg1) {
; OPT-NEXT: [[TMP2:%.*]] = shl nsw i32 [[ARG:%.*]], 1
; OPT-NEXT: br label [[LOOP:%.*]]
; OPT: loop:
-; OPT-NEXT: [[PHI_BROKEN1:%.*]] = phi i64 [ [[TMP7:%.*]], [[LOOP_END:%.*]] ], [ [[PHI_BROKEN1]], [[LOOP]] ], [ 0, [[ENTRY:%.*]] ]
-; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i64 [ 0, [[LOOP_END]] ], [ [[TMP0:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ]
+; OPT-NEXT: [[PHI_BROKEN1:%.*]] = phi i1 [ [[TMP7:%.*]], [[LOOP_END:%.*]] ], [ [[PHI_BROKEN1]], [[LOOP]] ], [ false, [[ENTRY:%.*]] ]
+; OPT-NEXT: [[PHI_BROKEN:%.*]] = phi i1 [ false, [[LOOP_END]] ], [ [[TMP0:%.*]], [[LOOP]] ], [ false, [[ENTRY]] ]
; OPT-NEXT: [[TMP4:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP5:%.*]], [[LOOP]] ], [ 0, [[LOOP_END]] ]
; OPT-NEXT: [[TMP5]] = add nsw i32 [[TMP4]], [[TMP]]
; OPT-NEXT: [[TMP6:%.*]] = icmp slt i32 [[ARG]], [[TMP5]]
-; OPT-NEXT: [[TMP0]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[TMP6]], i64 [[PHI_BROKEN]])
-; OPT-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP0]])
+; OPT-NEXT: [[TMP0]] = call i1 @llvm.amdgcn.if.break(i1 [[TMP6]], i1 [[PHI_BROKEN]])
+; OPT-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.loop(i1 [[TMP0]])
; OPT-NEXT: br i1 [[TMP1]], label [[LOOP_END]], label [[LOOP]]
; OPT: loop_end:
-; OPT-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP0]])
+; OPT-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP0]])
; OPT-NEXT: [[EXIT:%.*]] = icmp sgt i32 [[TMP5]], [[TMP2]]
-; OPT-NEXT: [[TMP7]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[EXIT]], i64 [[PHI_BROKEN1]])
-; OPT-NEXT: [[TMP3:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP7]])
+; OPT-NEXT: [[TMP7]] = call i1 @llvm.amdgcn.if.break(i1 [[EXIT]], i1 [[PHI_BROKEN1]])
+; OPT-NEXT: [[TMP3:%.*]] = call i1 @llvm.amdgcn.loop(i1 [[TMP7]])
; OPT-NEXT: br i1 [[TMP3]], label [[LOOP_EXIT:%.*]], label [[LOOP]]
; OPT: loop_exit:
-; OPT-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP7]])
+; OPT-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP7]])
; OPT-NEXT: [[TMP12:%.*]] = zext i32 [[TMP]] to i64
; OPT-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[ARG1:%.*]], i64 [[TMP12]]
; OPT-NEXT: [[TMP14:%.*]] = addrspacecast ptr [[TMP13]] to ptr addrspace(1)
diff --git a/llvm/test/CodeGen/AMDGPU/si-unify-exit-return-unreachable.ll b/llvm/test/CodeGen/AMDGPU/si-unify-exit-return-unreachable.ll
index 2d719bc777970..142fbe5a33bab 100644
--- a/llvm/test/CodeGen/AMDGPU/si-unify-exit-return-unreachable.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-unify-exit-return-unreachable.ll
@@ -28,9 +28,9 @@ define void @my_func(i32 %0) {
; IR: Flow11:
; IR-NEXT: [[TMP4:%.*]] = phi i1 [ [[TMP9:%.*]], [[FLOW12]] ], [ false, [[FLOW]] ]
; IR-NEXT: [[TMP5:%.*]] = phi i1 [ [[TMP10:%.*]], [[FLOW12]] ], [ [[TMP2]], [[FLOW]] ]
-; IR-NEXT: [[TMP6:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[TMP5]])
-; IR-NEXT: [[TMP7:%.*]] = extractvalue { i1, i64 } [[TMP6]], 0
-; IR-NEXT: [[TMP8:%.*]] = extractvalue { i1, i64 } [[TMP6]], 1
+; IR-NEXT: [[TMP6:%.*]] = call { i1, i1 } @llvm.amdgcn.if(i1 [[TMP5]])
+; IR-NEXT: [[TMP7:%.*]] = extractvalue { i1, i1 } [[TMP6]], 0
+; IR-NEXT: [[TMP8:%.*]] = extractvalue { i1, i1 } [[TMP6]], 1
; IR-NEXT: br i1 [[TMP7]], label [[DO_BODY:%.*]], label [[FLOW17:%.*]]
; IR: sw.bb2:
; IR-NEXT: br label [[NODEBLOCK7:%.*]]
@@ -40,9 +40,9 @@ define void @my_func(i32 %0) {
; IR-NEXT: br label [[FLOW11]]
; IR: NodeBlock7:
; IR-NEXT: [[PIVOT8:%.*]] = icmp sge i32 [[TMP0:%.*]], 2
-; IR-NEXT: [[TMP11:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[PIVOT8]])
-; IR-NEXT: [[TMP12:%.*]] = extractvalue { i1, i64 } [[TMP11]], 0
-; IR-NEXT: [[TMP13:%.*]] = extractvalue { i1, i64 } [[TMP11]], 1
+; IR-NEXT: [[TMP11:%.*]] = call { i1, i1 } @llvm.amdgcn.if(i1 [[PIVOT8]])
+; IR-NEXT: [[TMP12:%.*]] = extractvalue { i1, i1 } [[TMP11]], 0
+; IR-NEXT: [[TMP13:%.*]] = extractvalue { i1, i1 } [[TMP11]], 1
; IR-NEXT: br i1 [[TMP12]], label [[LEAFBLOCK5:%.*]], label [[FLOW13:%.*]]
; IR: LeafBlock5:
; IR-NEXT: [[SWITCHLEAF6:%.*]] = icmp eq i32 [[TMP0]], 2
@@ -50,9 +50,9 @@ define void @my_func(i32 %0) {
; IR: Flow13:
; IR-NEXT: [[TMP14:%.*]] = phi i1 [ true, [[LEAFBLOCK5]] ], [ false, [[NODEBLOCK7]] ]
; IR-NEXT: [[TMP15:%.*]] = phi i1 [ [[SWITCHLEAF6]], [[LEAFBLOCK5]] ], [ false, [[NODEBLOCK7]] ]
-; IR-NEXT: [[TMP16:%.*]] = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 [[TMP13]])
-; IR-NEXT: [[TMP17:%.*]] = extractvalue { i1, i64 } [[TMP16]], 0
-; IR-NEXT: [[TMP18:%.*]] = extractvalue { i1, i64 } [[TMP16]], 1
+; IR-NEXT: [[TMP16:%.*]] = call { i1, i1 } @llvm.amdgcn.else(i1 [[TMP13]])
+; IR-NEXT: [[TMP17:%.*]] = extractvalue { i1, i1 } [[TMP16]], 0
+; IR-NEXT: [[TMP18:%.*]] = extractvalue { i1, i1 } [[TMP16]], 1
; IR-NEXT: br i1 [[TMP17]], label [[LEAFBLOCK3:%.*]], label [[FLOW14:%.*]]
; IR: LeafBlock3:
; IR-NEXT: [[SWITCHLEAF4:%.*]] = icmp eq i32 [[TMP0]], 0
@@ -61,23 +61,23 @@ define void @my_func(i32 %0) {
; IR: Flow14:
; IR-NEXT: [[TMP19:%.*]] = phi i1 [ [[SWITCHLEAF4_INV]], [[LEAFBLOCK3]] ], [ [[TMP14]], [[FLOW13]] ]
; IR-NEXT: [[TMP20:%.*]] = phi i1 [ [[SWITCHLEAF4]], [[LEAFBLOCK3]] ], [ [[TMP15]], [[FLOW13]] ]
-; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP18]])
-; IR-NEXT: [[TMP21:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[TMP20]])
-; IR-NEXT: [[TMP22:%.*]] = extractvalue { i1, i64 } [[TMP21]], 0
-; IR-NEXT: [[TMP23:%.*]] = extractvalue { i1, i64 } [[TMP21]], 1
+; IR-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP18]])
+; IR-NEXT: [[TMP21:%.*]] = call { i1, i1 } @llvm.amdgcn.if(i1 [[TMP20]])
+; IR-NEXT: [[TMP22:%.*]] = extractvalue { i1, i1 } [[TMP21]], 0
+; IR-NEXT: [[TMP23:%.*]] = extractvalue { i1, i1 } [[TMP21]], 1
; IR-NEXT: br i1 [[TMP22]], label [[LAND_LHS_TRUE_I:%.*]], label [[FLOW15]]
; IR: land.lhs.true.i:
; IR-NEXT: br label [[LEAFBLOCK9:%.*]]
; IR: Flow15:
; IR-NEXT: [[TMP24]] = phi i1 [ [[TMP29:%.*]], [[FLOW16:%.*]] ], [ false, [[FLOW14]] ]
; IR-NEXT: [[TMP25]] = phi i1 [ [[TMP30:%.*]], [[FLOW16]] ], [ [[TMP19]], [[FLOW14]] ]
-; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP23]])
+; IR-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP23]])
; IR-NEXT: br label [[FLOW12]]
; IR: LeafBlock9:
; IR-NEXT: [[SWITCHLEAF10:%.*]] = icmp sgt i32 [[TMP0]], 1
-; IR-NEXT: [[TMP26:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[SWITCHLEAF10]])
-; IR-NEXT: [[TMP27:%.*]] = extractvalue { i1, i64 } [[TMP26]], 0
-; IR-NEXT: [[TMP28:%.*]] = extractvalue { i1, i64 } [[TMP26]], 1
+; IR-NEXT: [[TMP26:%.*]] = call { i1, i1 } @llvm.amdgcn.if(i1 [[SWITCHLEAF10]])
+; IR-NEXT: [[TMP27:%.*]] = extractvalue { i1, i1 } [[TMP26]], 0
+; IR-NEXT: [[TMP28:%.*]] = extractvalue { i1, i1 } [[TMP26]], 1
; IR-NEXT: br i1 [[TMP27]], label [[DO_BODY_I_I_I_I:%.*]], label [[FLOW16]]
; IR: do.body.i.i.i.i:
; IR-NEXT: tail call fastcc void null()
@@ -85,23 +85,23 @@ define void @my_func(i32 %0) {
; IR: Flow16:
; IR-NEXT: [[TMP29]] = phi i1 [ true, [[DO_BODY_I_I_I_I]] ], [ false, [[LEAFBLOCK9]] ]
; IR-NEXT: [[TMP30]] = phi i1 [ false, [[DO_BODY_I_I_I_I]] ], [ true, [[LEAFBLOCK9]] ]
-; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP28]])
+; IR-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP28]])
; IR-NEXT: br label [[FLOW15]]
; IR: do.body:
; IR-NEXT: tail call fastcc void null()
; IR-NEXT: br label [[FLOW17]]
; IR: Flow17:
; IR-NEXT: [[TMP31:%.*]] = phi i1 [ true, [[DO_BODY]] ], [ [[TMP4]], [[FLOW11]] ]
-; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP8]])
-; IR-NEXT: [[TMP32:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[TMP31]])
-; IR-NEXT: [[TMP33:%.*]] = extractvalue { i1, i64 } [[TMP32]], 0
-; IR-NEXT: [[TMP34:%.*]] = extractvalue { i1, i64 } [[TMP32]], 1
+; IR-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP8]])
+; IR-NEXT: [[TMP32:%.*]] = call { i1, i1 } @llvm.amdgcn.if(i1 [[TMP31]])
+; IR-NEXT: [[TMP33:%.*]] = extractvalue { i1, i1 } [[TMP32]], 0
+; IR-NEXT: [[TMP34:%.*]] = extractvalue { i1, i1 } [[TMP32]], 1
; IR-NEXT: br i1 [[TMP33]], label [[UNIFIEDUNREACHABLEBLOCK:%.*]], label [[UNIFIEDRETURNBLOCK:%.*]]
; IR: UnifiedUnreachableBlock:
; IR-NEXT: call void @llvm.amdgcn.unreachable()
; IR-NEXT: br label [[UNIFIEDRETURNBLOCK]]
; IR: UnifiedReturnBlock:
-; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP34]])
+; IR-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP34]])
; IR-NEXT: ret void
;
; GCN-LABEL: my_func:
diff --git a/llvm/test/CodeGen/AMDGPU/srem64.ll b/llvm/test/CodeGen/AMDGPU/srem64.ll
index 082620a68e1b5..6e7e81e485675 100644
--- a/llvm/test/CodeGen/AMDGPU/srem64.ll
+++ b/llvm/test/CodeGen/AMDGPU/srem64.ll
@@ -415,9 +415,9 @@ define i64 @v_test_srem(i64 %x, i64 %y) {
; GCN-IR-NEXT: v_add_i32_e32 v16, vcc, v6, v11
; GCN-IR-NEXT: v_lshr_b64 v[8:9], v[0:1], v8
; GCN-IR-NEXT: v_addc_u32_e64 v17, s[8:9], -1, 0, vcc
-; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v10, 0
; GCN-IR-NEXT: v_mov_b32_e32 v11, 0
+; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v7, 0
; GCN-IR-NEXT: .LBB1_3: ; %udiv-do-while
; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -1867,9 +1867,9 @@ define i64 @v_test_srem_k_num_i64(i64 %x) {
; GCN-IR-NEXT: v_sub_i32_e32 v12, vcc, 58, v8
; GCN-IR-NEXT: v_lshr_b64 v[6:7], 24, v6
; GCN-IR-NEXT: v_subb_u32_e64 v13, s[8:9], 0, 0, vcc
-; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v8, 0
; GCN-IR-NEXT: v_mov_b32_e32 v9, 0
+; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: .LBB11_3: ; %udiv-do-while
; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -2059,9 +2059,9 @@ define i64 @v_test_srem_pow2_k_num_i64(i64 %x) {
; GCN-IR-NEXT: v_sub_i32_e32 v12, vcc, 47, v8
; GCN-IR-NEXT: v_lshr_b64 v[6:7], s[8:9], v6
; GCN-IR-NEXT: v_subb_u32_e64 v13, s[8:9], 0, 0, vcc
-; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v8, 0
; GCN-IR-NEXT: v_mov_b32_e32 v9, 0
+; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: .LBB12_3: ; %udiv-do-while
; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -2161,9 +2161,9 @@ define i64 @v_test_srem_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_add_i32_e32 v12, vcc, 0xffffffcf, v8
; GCN-IR-NEXT: v_lshr_b64 v[6:7], v[0:1], v6
; GCN-IR-NEXT: v_addc_u32_e64 v13, s[8:9], 0, -1, vcc
-; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v8, 0
; GCN-IR-NEXT: v_mov_b32_e32 v9, 0
+; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: s_movk_i32 s10, 0x7fff
; GCN-IR-NEXT: .LBB13_3: ; %udiv-do-while
@@ -2510,9 +2510,9 @@ define i64 @v_test_srem24_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_add_i32_e32 v12, vcc, 0xffffffcf, v8
; GCN-IR-NEXT: v_lshr_b64 v[6:7], v[0:1], v6
; GCN-IR-NEXT: v_addc_u32_e64 v13, s[8:9], 0, -1, vcc
-; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v8, 0
; GCN-IR-NEXT: v_mov_b32_e32 v9, 0
+; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: s_movk_i32 s10, 0x7fff
; GCN-IR-NEXT: .LBB18_3: ; %udiv-do-while
diff --git a/llvm/test/CodeGen/AMDGPU/udiv64.ll b/llvm/test/CodeGen/AMDGPU/udiv64.ll
index a37f09722d39d..c1e0e7416d51e 100644
--- a/llvm/test/CodeGen/AMDGPU/udiv64.ll
+++ b/llvm/test/CodeGen/AMDGPU/udiv64.ll
@@ -354,9 +354,9 @@ define i64 @v_test_udiv_i64(i64 %x, i64 %y) {
; GCN-IR-NEXT: v_not_b32_e32 v6, v8
; GCN-IR-NEXT: v_add_i32_e32 v12, vcc, v6, v9
; GCN-IR-NEXT: v_addc_u32_e64 v13, s[8:9], -1, 0, vcc
-; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v8, 0
; GCN-IR-NEXT: v_mov_b32_e32 v9, 0
+; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v7, 0
; GCN-IR-NEXT: .LBB1_3: ; %udiv-do-while
; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -1215,9 +1215,9 @@ define i64 @v_test_udiv_pow2_k_num_i64(i64 %x) {
; GCN-IR-NEXT: v_sub_i32_e32 v12, vcc, 47, v8
; GCN-IR-NEXT: v_lshr_b64 v[6:7], s[8:9], v6
; GCN-IR-NEXT: v_subb_u32_e64 v13, s[8:9], 0, 0, vcc
-; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v8, 0
; GCN-IR-NEXT: v_mov_b32_e32 v9, 0
+; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: .LBB9_3: ; %udiv-do-while
; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -1300,9 +1300,9 @@ define i64 @v_test_udiv_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_add_i32_e32 v8, vcc, 0xffffffcf, v6
; GCN-IR-NEXT: v_lshr_b64 v[0:1], v[0:1], v7
; GCN-IR-NEXT: v_addc_u32_e64 v9, s[8:9], 0, -1, vcc
-; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v6, 0
; GCN-IR-NEXT: v_mov_b32_e32 v7, 0
+; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: s_movk_i32 s10, 0x7fff
; GCN-IR-NEXT: .LBB10_3: ; %udiv-do-while
@@ -1503,9 +1503,9 @@ define i64 @v_test_udiv_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_add_i32_e32 v8, vcc, 0xffffffc4, v6
; GCN-IR-NEXT: v_lshr_b64 v[0:1], v[0:1], v7
; GCN-IR-NEXT: v_addc_u32_e64 v9, s[8:9], 0, -1, vcc
-; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v6, 0
; GCN-IR-NEXT: v_mov_b32_e32 v7, 0
+; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: .LBB12_3: ; %udiv-do-while
; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -1838,9 +1838,9 @@ define i64 @v_test_udiv24_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_add_i32_e32 v8, vcc, 0xffffffcf, v6
; GCN-IR-NEXT: v_lshr_b64 v[4:5], v[4:5], v7
; GCN-IR-NEXT: v_addc_u32_e64 v9, s[8:9], 0, -1, vcc
-; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v6, 0
; GCN-IR-NEXT: v_mov_b32_e32 v7, 0
+; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v3, 0
; GCN-IR-NEXT: s_movk_i32 s10, 0x7fff
; GCN-IR-NEXT: .LBB17_3: ; %udiv-do-while
diff --git a/llvm/test/CodeGen/AMDGPU/unstructured-cfg-def-use-issue.ll b/llvm/test/CodeGen/AMDGPU/unstructured-cfg-def-use-issue.ll
index fa0bdd8ad9ac9..75c56d7ac2fb4 100644
--- a/llvm/test/CodeGen/AMDGPU/unstructured-cfg-def-use-issue.ll
+++ b/llvm/test/CodeGen/AMDGPU/unstructured-cfg-def-use-issue.ll
@@ -154,12 +154,12 @@ define hidden void @widget() #0 {
; SI-OPT: bb9:
; SI-OPT-NEXT: [[TMP10:%.*]] = call float @wibble()
; SI-OPT-NEXT: [[TMP11:%.*]] = fcmp nsz ogt float [[TMP10]], 0.000000e+00
-; SI-OPT-NEXT: [[TMP0:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[TMP11]])
-; SI-OPT-NEXT: [[TMP1:%.*]] = extractvalue { i1, i64 } [[TMP0]], 0
-; SI-OPT-NEXT: [[TMP2:%.*]] = extractvalue { i1, i64 } [[TMP0]], 1
+; SI-OPT-NEXT: [[TMP0:%.*]] = call { i1, i1 } @llvm.amdgcn.if(i1 [[TMP11]])
+; SI-OPT-NEXT: [[TMP1:%.*]] = extractvalue { i1, i1 } [[TMP0]], 0
+; SI-OPT-NEXT: [[TMP2:%.*]] = extractvalue { i1, i1 } [[TMP0]], 1
; SI-OPT-NEXT: br i1 [[TMP1]], label [[BB6:%.*]], label [[BB9_BB12_CRIT_EDGE:%.*]]
; SI-OPT: bb9.bb12_crit_edge:
-; SI-OPT-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP2]])
+; SI-OPT-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP2]])
; SI-OPT-NEXT: br label [[BB12]]
; SI-OPT: bb12:
; SI-OPT-NEXT: store float 0.000000e+00, ptr addrspace(1) null, align 8
@@ -212,22 +212,22 @@ define hidden void @blam() #0 {
; SI-OPT-NEXT: br label [[BB4:%.*]]
; SI-OPT: bb4:
; SI-OPT-NEXT: [[TMP5:%.*]] = icmp slt i32 [[TMP3]], 3
-; SI-OPT-NEXT: [[TMP0:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[TMP5]])
-; SI-OPT-NEXT: [[TMP1:%.*]] = extractvalue { i1, i64 } [[TMP0]], 0
-; SI-OPT-NEXT: [[TMP2:%.*]] = extractvalue { i1, i64 } [[TMP0]], 1
+; SI-OPT-NEXT: [[TMP0:%.*]] = call { i1, i1 } @llvm.amdgcn.if(i1 [[TMP5]])
+; SI-OPT-NEXT: [[TMP1:%.*]] = extractvalue { i1, i1 } [[TMP0]], 0
+; SI-OPT-NEXT: [[TMP2:%.*]] = extractvalue { i1, i1 } [[TMP0]], 1
; SI-OPT-NEXT: br i1 [[TMP1]], label [[BB8:%.*]], label [[BB6:%.*]]
; SI-OPT: bb6:
-; SI-OPT-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP2]])
+; SI-OPT-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP2]])
; SI-OPT-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP3]], 3
; SI-OPT-NEXT: br i1 [[TMP7]], label [[BB11:%.*]], label [[BB1:%.*]]
; SI-OPT: bb8:
; SI-OPT-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP3]], 1
-; SI-OPT-NEXT: [[TMP3:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[TMP9]])
-; SI-OPT-NEXT: [[TMP4:%.*]] = extractvalue { i1, i64 } [[TMP3]], 0
-; SI-OPT-NEXT: [[TMP5:%.*]] = extractvalue { i1, i64 } [[TMP3]], 1
+; SI-OPT-NEXT: [[TMP3:%.*]] = call { i1, i1 } @llvm.amdgcn.if(i1 [[TMP9]])
+; SI-OPT-NEXT: [[TMP4:%.*]] = extractvalue { i1, i1 } [[TMP3]], 0
+; SI-OPT-NEXT: [[TMP5:%.*]] = extractvalue { i1, i1 } [[TMP3]], 1
; SI-OPT-NEXT: br i1 [[TMP4]], label [[BB10:%.*]], label [[BB8_BB1_CRIT_EDGE:%.*]]
; SI-OPT: bb8.bb1_crit_edge:
-; SI-OPT-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP5]])
+; SI-OPT-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP5]])
; SI-OPT-NEXT: br label [[BB1]]
; SI-OPT: bb10:
; SI-OPT-NEXT: store float +qnan, ptr addrspace(5) null, align 16
@@ -235,19 +235,19 @@ define hidden void @blam() #0 {
; SI-OPT: bb11:
; SI-OPT-NEXT: [[TMP12:%.*]] = call float @spam()
; SI-OPT-NEXT: [[TMP13:%.*]] = fcmp nsz oeq float [[TMP12]], 0.000000e+00
-; SI-OPT-NEXT: [[TMP6:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[TMP13]])
-; SI-OPT-NEXT: [[TMP7:%.*]] = extractvalue { i1, i64 } [[TMP6]], 0
-; SI-OPT-NEXT: [[TMP8:%.*]] = extractvalue { i1, i64 } [[TMP6]], 1
+; SI-OPT-NEXT: [[TMP6:%.*]] = call { i1, i1 } @llvm.amdgcn.if(i1 [[TMP13]])
+; SI-OPT-NEXT: [[TMP7:%.*]] = extractvalue { i1, i1 } [[TMP6]], 0
+; SI-OPT-NEXT: [[TMP8:%.*]] = extractvalue { i1, i1 } [[TMP6]], 1
; SI-OPT-NEXT: br i1 [[TMP7]], label [[BB2]], label [[BB14:%.*]]
; SI-OPT: bb14:
-; SI-OPT-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP8]])
+; SI-OPT-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP8]])
; SI-OPT-NEXT: [[TMP15:%.*]] = fcmp nsz oeq float [[TMP]], 0.000000e+00
-; SI-OPT-NEXT: [[TMP9:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[TMP15]])
-; SI-OPT-NEXT: [[TMP10:%.*]] = extractvalue { i1, i64 } [[TMP9]], 0
-; SI-OPT-NEXT: [[TMP11:%.*]] = extractvalue { i1, i64 } [[TMP9]], 1
+; SI-OPT-NEXT: [[TMP9:%.*]] = call { i1, i1 } @llvm.amdgcn.if(i1 [[TMP15]])
+; SI-OPT-NEXT: [[TMP10:%.*]] = extractvalue { i1, i1 } [[TMP9]], 0
+; SI-OPT-NEXT: [[TMP11:%.*]] = extractvalue { i1, i1 } [[TMP9]], 1
; SI-OPT-NEXT: br i1 [[TMP10]], label [[BB17:%.*]], label [[BB16:%.*]]
; SI-OPT: bb16:
-; SI-OPT-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 [[TMP11]])
+; SI-OPT-NEXT: call void @llvm.amdgcn.end.cf(i1 [[TMP11]])
; SI-OPT-NEXT: store float +qnan, ptr addrspace(5) null, align 16
; SI-OPT-NEXT: br label [[BB17]]
; SI-OPT: bb17:
@@ -329,8 +329,8 @@ define hidden void @blam() #0 {
; GCN-NEXT: .LBB1_2: ; %bb2
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN-NEXT: flat_load_dword v0, v[41:42]
-; GCN-NEXT: s_mov_b64 s[6:7], 0
; GCN-NEXT: s_mov_b64 s[4:5], -1
+; GCN-NEXT: s_mov_b64 s[6:7], 0
; GCN-NEXT: buffer_store_dword v42, off, s[0:3], 0
; GCN-NEXT: s_waitcnt vmcnt(1)
; GCN-NEXT: v_cmp_lt_i32_e32 vcc, 2, v0
diff --git a/llvm/test/CodeGen/AMDGPU/urem64.ll b/llvm/test/CodeGen/AMDGPU/urem64.ll
index 52235796d7089..0b9df5c15fc26 100644
--- a/llvm/test/CodeGen/AMDGPU/urem64.ll
+++ b/llvm/test/CodeGen/AMDGPU/urem64.ll
@@ -390,9 +390,9 @@ define i64 @v_test_urem_i64(i64 %x, i64 %y) {
; GCN-IR-NEXT: v_add_i32_e32 v14, vcc, v6, v11
; GCN-IR-NEXT: v_lshr_b64 v[8:9], v[0:1], v8
; GCN-IR-NEXT: v_addc_u32_e64 v15, s[8:9], -1, 0, vcc
-; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v10, 0
; GCN-IR-NEXT: v_mov_b32_e32 v11, 0
+; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v7, 0
; GCN-IR-NEXT: .LBB1_3: ; %udiv-do-while
; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -1295,9 +1295,9 @@ define i64 @v_test_urem_pow2_k_num_i64(i64 %x) {
; GCN-IR-NEXT: v_sub_i32_e32 v12, vcc, 47, v8
; GCN-IR-NEXT: v_lshr_b64 v[6:7], s[8:9], v6
; GCN-IR-NEXT: v_subb_u32_e64 v13, s[8:9], 0, 0, vcc
-; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v8, 0
; GCN-IR-NEXT: v_mov_b32_e32 v9, 0
+; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: .LBB8_3: ; %udiv-do-while
; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -1386,9 +1386,9 @@ define i64 @v_test_urem_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_add_i32_e32 v10, vcc, 0xffffffcf, v8
; GCN-IR-NEXT: v_lshr_b64 v[6:7], v[0:1], v6
; GCN-IR-NEXT: v_addc_u32_e64 v11, s[8:9], 0, -1, vcc
-; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v8, 0
; GCN-IR-NEXT: v_mov_b32_e32 v9, 0
+; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: s_movk_i32 s10, 0x7fff
; GCN-IR-NEXT: .LBB9_3: ; %udiv-do-while
@@ -1699,9 +1699,9 @@ define i64 @v_test_urem24_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_lshr_b64 v[6:7], v[0:1], v6
; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 0xffffffcf, v8
; GCN-IR-NEXT: v_addc_u32_e64 v10, s[8:9], 0, -1, vcc
-; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v8, 0
; GCN-IR-NEXT: v_mov_b32_e32 v9, 0
+; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: s_movk_i32 s10, 0x7fff
; GCN-IR-NEXT: .LBB14_3: ; %udiv-do-while
diff --git a/llvm/test/CodeGen/AMDGPU/valu-i1.ll b/llvm/test/CodeGen/AMDGPU/valu-i1.ll
index 6f20a4e290c90..e6b1d885eef14 100644
--- a/llvm/test/CodeGen/AMDGPU/valu-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/valu-i1.ll
@@ -324,16 +324,16 @@ define amdgpu_kernel void @multi_vcond_loop(ptr addrspace(1) noalias nocapture %
; SI-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; SI-NEXT: v_addc_u32_e32 v7, vcc, 0, v7, vcc
; SI-NEXT: s_mov_b64 s[2:3], 0
+; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_mov_b32 s8, s10
; SI-NEXT: s_mov_b32 s9, s10
-; SI-NEXT: s_mov_b64 s[6:7], 0
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
+; SI-NEXT: ; implicit-def: $sgpr6_sgpr7
; SI-NEXT: .LBB5_2: ; %bb10
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v8, v[6:7], s[8:11], 0 addr64
; SI-NEXT: buffer_load_dword v9, v[4:5], s[8:11], 0 addr64
-; SI-NEXT: s_or_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_or_b64 s[6:7], s[6:7], exec
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, -1, v8
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -349,20 +349,20 @@ define amdgpu_kernel void @multi_vcond_loop(ptr addrspace(1) noalias nocapture %
; SI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; SI-NEXT: v_add_i32_e32 v4, vcc, 4, v4
; SI-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; SI-NEXT: s_add_u32 s6, s6, 1
+; SI-NEXT: s_add_u32 s2, s2, 1
; SI-NEXT: v_add_i32_e32 v6, vcc, 4, v6
-; SI-NEXT: s_addc_u32 s7, s7, 0
+; SI-NEXT: s_addc_u32 s3, s3, 0
; SI-NEXT: v_addc_u32_e32 v7, vcc, 0, v7, vcc
-; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1]
-; SI-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
+; SI-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
; SI-NEXT: s_and_b64 s[12:13], vcc, exec
-; SI-NEXT: s_or_b64 s[4:5], s[4:5], s[12:13]
+; SI-NEXT: s_or_b64 s[6:7], s[6:7], s[12:13]
; SI-NEXT: .LBB5_4: ; %Flow
; SI-NEXT: ; in Loop: Header=BB5_2 Depth=1
; SI-NEXT: s_or_b64 exec, exec, s[0:1]
-; SI-NEXT: s_and_b64 s[0:1], exec, s[4:5]
-; SI-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
-; SI-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; SI-NEXT: s_and_b64 s[0:1], exec, s[6:7]
+; SI-NEXT: s_or_b64 s[4:5], s[0:1], s[4:5]
+; SI-NEXT: s_andn2_b64 exec, exec, s[4:5]
; SI-NEXT: s_cbranch_execnz .LBB5_2
; SI-NEXT: .LBB5_5: ; %bb26
; SI-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/wave32.ll b/llvm/test/CodeGen/AMDGPU/wave32.ll
index d857d147a248d..aac2f8ad7aaab 100644
--- a/llvm/test/CodeGen/AMDGPU/wave32.ll
+++ b/llvm/test/CodeGen/AMDGPU/wave32.ll
@@ -532,9 +532,9 @@ define amdgpu_kernel void @test_loop_with_if_else_break(ptr addrspace(1) %arg) #
; GFX1032-NEXT: s_branch .LBB11_4
; GFX1032-NEXT: .LBB11_2: ; %bb8
; GFX1032-NEXT: ; in Loop: Header=BB11_4 Depth=1
-; GFX1032-NEXT: s_add_i32 s3, s3, 1
+; GFX1032-NEXT: s_add_i32 s2, s2, 1
; GFX1032-NEXT: global_store_dword v2, v0, s[0:1]
-; GFX1032-NEXT: v_cmp_ge_u32_e32 vcc_lo, s3, v1
+; GFX1032-NEXT: v_cmp_ge_u32_e32 vcc_lo, s2, v1
; GFX1032-NEXT: s_add_u32 s0, s0, 4
; GFX1032-NEXT: s_addc_u32 s1, s1, 0
; GFX1032-NEXT: s_andn2_b32 s4, s4, exec_lo
@@ -543,8 +543,8 @@ define amdgpu_kernel void @test_loop_with_if_else_break(ptr addrspace(1) %arg) #
; GFX1032-NEXT: .LBB11_3: ; %Flow
; GFX1032-NEXT: ; in Loop: Header=BB11_4 Depth=1
; GFX1032-NEXT: s_and_b32 s5, exec_lo, s4
-; GFX1032-NEXT: s_or_b32 s2, s5, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, s5, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execz .LBB11_6
; GFX1032-NEXT: .LBB11_4: ; %bb2
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -555,7 +555,7 @@ define amdgpu_kernel void @test_loop_with_if_else_break(ptr addrspace(1) %arg) #
; GFX1032-NEXT: v_cmp_gt_i32_e32 vcc_lo, 11, v3
; GFX1032-NEXT: s_cbranch_vccz .LBB11_2
; GFX1032-NEXT: ; %bb.5: ; in Loop: Header=BB11_4 Depth=1
-; GFX1032-NEXT: ; implicit-def: $sgpr3
+; GFX1032-NEXT: ; implicit-def: $sgpr2
; GFX1032-NEXT: ; implicit-def: $sgpr0_sgpr1
; GFX1032-NEXT: s_branch .LBB11_3
; GFX1032-NEXT: .LBB11_6: ; %.loopexit
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/syncscopes.mir b/llvm/test/CodeGen/MIR/AMDGPU/syncscopes.mir
index c465d7fc77bed..a2afe120bcb4a 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/syncscopes.mir
+++ b/llvm/test/CodeGen/MIR/AMDGPU/syncscopes.mir
@@ -15,22 +15,22 @@
}
; Function Attrs: convergent nounwind
- declare { i1, i64 } @llvm.amdgcn.if(i1) #1
+ declare { i1, i1 } @llvm.amdgcn.if(i1) #1
; Function Attrs: convergent nounwind
- declare { i1, i64 } @llvm.amdgcn.else(i64) #1
+ declare { i1, i1 } @llvm.amdgcn.else(i1) #1
; Function Attrs: convergent nounwind readnone
declare i64 @llvm.amdgcn.break(i64) #2
; Function Attrs: convergent nounwind readnone
- declare i64 @llvm.amdgcn.if.break(i1, i64) #2
+ declare i1 @llvm.amdgcn.if.break(i1, i1) #2
; Function Attrs: convergent nounwind
- declare i1 @llvm.amdgcn.loop(i64) #1
+ declare i1 @llvm.amdgcn.loop(i1) #1
; Function Attrs: convergent nounwind
- declare void @llvm.amdgcn.end.cf(i64) #1
+ declare void @llvm.amdgcn.end.cf(i1) #1
attributes #0 = { "target-cpu"="gfx803" }
attributes #1 = { convergent nounwind }
More information about the llvm-commits
mailing list